Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Daily Mirrot and The Sun

Browse Source
This commit is contained in:
Kovid Goyal 2012-05-01 09:17:20 +05:30
parent 33b0bc0601
commit 48214035de
2 changed files with 55 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
recipes/daily_mirror.recipe
View File

@ -1,13 +1,13 @@
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
from claibre import browser
import re import re
import mechanize
class AdvancedUserRecipe1306061239(BasicNewsRecipe): class AdvancedUserRecipe1306061239(BasicNewsRecipe):
title = u'The Daily Mirror' title = u'The Daily Mirror'
description = 'News as provide by The Daily Mirror -UK' description = 'News as provided by The Daily Mirror -UK'
__author__ = 'Dave Asbury' __author__ = 'Dave Asbury'
# last updated 7/4/12 # last updated 28/4/12
language = 'en_GB' language = 'en_GB'
#cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' #cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg'
@ -15,89 +15,80 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe):
oldest_article = 1 oldest_article = 1
max_articles_per_feed = 10 max_articles_per_feed = 12
remove_empty_feeds = True remove_empty_feeds = True
remove_javascript = True remove_javascript = True
no_stylesheets = True no_stylesheets = True
auto_cleanup = True # auto_cleanup = True
#conversion_options = { 'linearize_tables' : True } #conversion_options = { 'linearize_tables' : True }
#keep_only_tags = [
# dict(name='h1'), keep_only_tags = [ dict(name='h1'),
# dict(name='div',attrs={'id' : 'body-content'}), dict(name='div',attrs={'class' : 'lead-text'}),
#dict(name='div',atts={'class' : 'article-body'}), dict(name='div',attrs={'class' : 'styleGroup clearfix'}),
dict(name='div',attrs={'class' : 'widget relatedContents pictures widget-editable viziwyg-section-245 inpage-widget-158123'}),
dict(name='figure',attrs={'class' : 'clearfix'}),
dict(name='div',attrs={'class' :'body '}),
#dict(attrs={'class' : ['article-attr','byline append-1','published']}), #dict(attrs={'class' : ['article-attr','byline append-1','published']}),
#dict(name='p'), #dict(name='p'),
# ] ]
#remove_tags_after = [dict (name='div',attrs={'class' : 'related'})]
remove_tags = [ remove_tags = [
dict(attrs={'class' : 'comment'}),
dict(name='title'), dict(name='title'),
dict(name='div',attrs={'class' : ['inline-ad span-16 last','caption']}), dict(name='ul',attrs={'class' : 'clearfix breadcrumbs '}),
# dict(name='div',attrs={'id' : ['sidebar','menu','search-box','roffers-top']}), dict(name='ul',attrs={'id' : 'login-201109171215'}),
#dict(name='div',attrs={'class' :['inline-ad span-16 last','article-resize','related','list teasers']}), dict(name='div',attrs={'class' : ['inline-ad span-16 last','caption']}),#'widget navigation breadcrumb widget-editable viziwyg-section-198 inpage-widget-80721 span-17','image-credit'
#dict(attrs={'class' : ['channellink','article-tags','replace','append-html']}), ]
]
# preprocess_regexps = [
#(re.compile(r'<dl class="q-search">.*?</dl>', re.IGNORECASE | re.DOTALL), lambda match: '')]
preprocess_regexps = [
(re.compile(r'- mirror.co.uk', re.IGNORECASE | re.DOTALL), lambda match: '')]
preprocess_regexps = [ preprocess_regexps = [
(re.compile(r'Advertisement >>', re.IGNORECASE | re.DOTALL), lambda match: '')] (re.compile(r'- mirror.co.uk', re.IGNORECASE | re.DOTALL), lambda match: '')]
#preprocess_regexps = [
#(re.compile(r'Sponsored Links', re.IGNORECASE | re.DOTALL), lambda match: '')]
feeds = [ feeds = [
(u'News',u'http://www.mirror.co.uk/news/rss.xml'),
(u'Sports',u'http://www.mirror.co.uk/sport/rss.xml'),
(u'3AM',u'http://www.mirror.co.uk/3am/rss.xml'),
(u'Lifestyle',u'http://www.mirror.co.uk/lifestyle/rss.xml')
(u'UK News', u'http://feed43.com/0287771688643868.xml')
,(u'Tech News', u'http://feed43.com/2455520588350501.xml')
,(u'Weird World','http://feed43.com/0863800333634654.xml')
,(u'Sport','http://feed43.com/7713243036546130.xml')
,(u'Sport : Boxing ','http://feed43.com/0414732220804255.xml')
,(u'Sport : Rugby Union','http://feed43.com/4710138762362383.xml')
,(u'Sport : Other','http://feed43.com/4501416886323415.xml')
,(u'TV and Film','http://feed43.com/5238302853765104.xml')
,(u'Celebs','http://feed43.com/8770061048844683.xml')
,(u'Life Style : Family','http://feed43.com/4356170742410338.xml')
,(u'Travel','http://feed43.com/1436576006476607.xml')
# example of commented out feed not needed ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml') # example of commented out feed not needed ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml')
] ]
extra_css = ''' extra_css = '''
body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} h1{ font-size:medium;}
h1{ font-size:18px;} body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
img { display:block} img { display:block}
''' '''#
def get_cover_url(self): def get_cover_url(self):
soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html') soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html')
# look for the block containing the mirror button and url # look for the block containing the mirror button and url
cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_92.gif);'}) cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_92.gif);'})
cov2 = str(cov) cov2 = str(cov)
cov2='http://www.politicshome.com'+cov2[9:-142] cov2='http://www.politicshome.com'+cov2[9:-142]
#cov2 now contains url of the page containing pic #cov2 now contains url of the page containing pic
soup = self.index_to_soup(cov2) soup = self.index_to_soup(cov2)
cov = soup.find(attrs={'id' : 'large'}) cov = soup.find(attrs={'id' : 'large'})
cov2 = str(cov) cov2 = str(cov)
cov2=cov2[27:-18] cov2=cov2[27:-18]
#cov2 now is pic url, now go back to original function #cov2 now is pic url, now go back to original function
br = mechanize.Browser() br = browser()
br.set_handle_redirect(False) br.set_handle_redirect(False)
try: try:
br.open_novisit(cov2) br.open_novisit(cov2)
cover_url = cov2 cover_url = cov2
except: except:
cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' cover_url ='http://profile.ak.fbcdn.net/hprofile-ak-snc4/373019_6149699161_1710984811_n.jpg'
# print '******** string is ', cov2,' ***'
#cover_url = cov2 #cover_url = cov2
#cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png' #cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'
return cover_url return cover_url

34
recipes/the_sun.recipe
View File

@ -1,12 +1,14 @@
import re, mechanize import re, random
from calibre import browser
from calibre.web.feeds.recipes import BasicNewsRecipe from calibre.web.feeds.recipes import BasicNewsRecipe
class AdvancedUserRecipe1325006965(BasicNewsRecipe): class AdvancedUserRecipe1325006965(BasicNewsRecipe):
title = u'The Sun UK' title = u'The Sun UK'
description = 'A Recipe for The Sun tabloid UK' description = 'A Recipe for The Sun tabloid UK'
__author__ = 'Dave Asbury' __author__ = 'Dave Asbury'
# last updated 7/4/12 # last updated 29/4/12
language = 'en_GB' language = 'en_GB'
oldest_article = 1 oldest_article = 1
max_articles_per_feed = 15 max_articles_per_feed = 15
@ -48,12 +50,10 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
feeds = [ feeds = [
(u'News','http://feed43.com/2517447382644748.xml'), (u'News', u'http://www.thesun.co.uk/sol/homepage/news/rss'),
(u'Sport', u'http://feed43.com/4283846255668687.xml'), (u'Sport', u'http://www.thesun.co.uk/sol/homepage/sport/rss'),
(u'Bizarre', u'http://feed43.com/0233840304242011.xml'), (u'Showbiz', u'http://www.thesun.co.uk/sol/homepage/showbiz/rss'),
(u'Film',u'http://feed43.com/1307545221226200.xml'), (u'Woman', u'http://www.thesun.co.uk/sol/homepage/woman/rss'),
(u'Music',u'http://feed43.com/1701513435064132.xml'),
(u'Sun Woman',u'http://feed43.com/0022626854226453.xml'),
] ]
def get_cover_url(self): def get_cover_url(self):
@ -61,14 +61,11 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
# look for the block containing the sun button and url # look for the block containing the sun button and url
cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_84.gif);'}) cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_84.gif);'})
#cov = soup.find(attrs={'id' : 'large'}) #cov = soup.find(attrs={'id' : 'large'})
cov2 = str(cov) cov2 = str(cov)
cov2='http://www.politicshome.com'+cov2[9:-133] cov2='http://www.politicshome.com'+cov2[9:-133]
#cov2 now contains url of the page containing pic #cov2 now contains url of the page containing pic
#cov2 now contains url of the page containing pic #cov2 now contains url of the page containing pic
soup = self.index_to_soup(cov2) soup = self.index_to_soup(cov2)
cov = soup.find(attrs={'id' : 'large'}) cov = soup.find(attrs={'id' : 'large'})
@ -76,16 +73,21 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
cov2=cov2[27:-18] cov2=cov2[27:-18]
#cov2 now is pic url, now go back to original function #cov2 now is pic url, now go back to original function
br = mechanize.Browser() br = browser()
br.set_handle_redirect(False) br.set_handle_redirect(False)
try: try:
br.open_novisit(cov2) br.open_novisit(cov2)
cover_url = cov2 cover_url = cov2
except: except:
cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png' cover_url = random.choice((
'http://img.thesun.co.uk/multimedia/archive/00905/errorpage6_677961a_905507a.jpg'
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage7_677962a_905505a.jpg'
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage5_677960a_905512a.jpg'
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage2_677957a_905502a.jpg'
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage3_677958a_905503a.jpg'
))
#cover_url = cov2
#cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'
return cover_url return cover_url