mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Update Daily Mirrot and The Sun
This commit is contained in:
Kovid Goyal
parent
33b0bc0601
commit
48214035de
@ -1,13 +1,13 @@
|
||||
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
from claibre import browser
|
||||
import re
|
||||
import mechanize
|
||||
|
||||
class AdvancedUserRecipe1306061239(BasicNewsRecipe):
|
||||
title = u'The Daily Mirror'
|
||||
description = 'News as provide by The Daily Mirror -UK'
|
||||
description = 'News as provided by The Daily Mirror -UK'
|
||||
|
||||
__author__ = 'Dave Asbury'
|
||||
# last updated 7/4/12
|
||||
# last updated 28/4/12
|
||||
language = 'en_GB'
|
||||
#cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg'
|
||||
|
||||
@ -15,66 +15,55 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe):
|
||||
|
||||
|
||||
oldest_article = 1
|
||||
max_articles_per_feed = 10
|
||||
max_articles_per_feed = 12
|
||||
remove_empty_feeds = True
|
||||
remove_javascript = True
|
||||
no_stylesheets = True
|
||||
auto_cleanup = True
|
||||
# auto_cleanup = True
|
||||
#conversion_options = { 'linearize_tables' : True }
|
||||
|
||||
|
||||
#keep_only_tags = [
|
||||
# dict(name='h1'),
|
||||
# dict(name='div',attrs={'id' : 'body-content'}),
|
||||
#dict(name='div',atts={'class' : 'article-body'}),
|
||||
|
||||
keep_only_tags = [ dict(name='h1'),
|
||||
dict(name='div',attrs={'class' : 'lead-text'}),
|
||||
dict(name='div',attrs={'class' : 'styleGroup clearfix'}),
|
||||
dict(name='div',attrs={'class' : 'widget relatedContents pictures widget-editable viziwyg-section-245 inpage-widget-158123'}),
|
||||
dict(name='figure',attrs={'class' : 'clearfix'}),
|
||||
dict(name='div',attrs={'class' :'body '}),
|
||||
|
||||
#dict(attrs={'class' : ['article-attr','byline append-1','published']}),
|
||||
#dict(name='p'),
|
||||
# ]
|
||||
|
||||
#remove_tags_after = [dict (name='div',attrs={'class' : 'related'})]
|
||||
|
||||
remove_tags = [
|
||||
dict(name='title'),
|
||||
dict(name='div',attrs={'class' : ['inline-ad span-16 last','caption']}),
|
||||
# dict(name='div',attrs={'id' : ['sidebar','menu','search-box','roffers-top']}),
|
||||
#dict(name='div',attrs={'class' :['inline-ad span-16 last','article-resize','related','list teasers']}),
|
||||
#dict(attrs={'class' : ['channellink','article-tags','replace','append-html']}),
|
||||
]
|
||||
|
||||
# preprocess_regexps = [
|
||||
#(re.compile(r'<dl class="q-search">.*?</dl>', re.IGNORECASE | re.DOTALL), lambda match: '')]
|
||||
|
||||
remove_tags = [
|
||||
dict(attrs={'class' : 'comment'}),
|
||||
dict(name='title'),
|
||||
dict(name='ul',attrs={'class' : 'clearfix breadcrumbs '}),
|
||||
dict(name='ul',attrs={'id' : 'login-201109171215'}),
|
||||
dict(name='div',attrs={'class' : ['inline-ad span-16 last','caption']}),#'widget navigation breadcrumb widget-editable viziwyg-section-198 inpage-widget-80721 span-17','image-credit'
|
||||
]
|
||||
|
||||
preprocess_regexps = [
|
||||
(re.compile(r'- mirror.co.uk', re.IGNORECASE | re.DOTALL), lambda match: '')]
|
||||
|
||||
preprocess_regexps = [
|
||||
(re.compile(r'Advertisement >>', re.IGNORECASE | re.DOTALL), lambda match: '')]
|
||||
|
||||
#preprocess_regexps = [
|
||||
#(re.compile(r'Sponsored Links', re.IGNORECASE | re.DOTALL), lambda match: '')]
|
||||
|
||||
feeds = [
|
||||
(u'News',u'http://www.mirror.co.uk/news/rss.xml'),
|
||||
(u'Sports',u'http://www.mirror.co.uk/sport/rss.xml'),
|
||||
(u'3AM',u'http://www.mirror.co.uk/3am/rss.xml'),
|
||||
(u'Lifestyle',u'http://www.mirror.co.uk/lifestyle/rss.xml')
|
||||
|
||||
(u'UK News', u'http://feed43.com/0287771688643868.xml')
|
||||
,(u'Tech News', u'http://feed43.com/2455520588350501.xml')
|
||||
,(u'Weird World','http://feed43.com/0863800333634654.xml')
|
||||
,(u'Sport','http://feed43.com/7713243036546130.xml')
|
||||
,(u'Sport : Boxing ','http://feed43.com/0414732220804255.xml')
|
||||
,(u'Sport : Rugby Union','http://feed43.com/4710138762362383.xml')
|
||||
,(u'Sport : Other','http://feed43.com/4501416886323415.xml')
|
||||
,(u'TV and Film','http://feed43.com/5238302853765104.xml')
|
||||
,(u'Celebs','http://feed43.com/8770061048844683.xml')
|
||||
,(u'Life Style : Family','http://feed43.com/4356170742410338.xml')
|
||||
,(u'Travel','http://feed43.com/1436576006476607.xml')
|
||||
|
||||
|
||||
|
||||
# example of commented out feed not needed ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml')
|
||||
]
|
||||
extra_css = '''
|
||||
h1{ font-size:medium;}
|
||||
body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
|
||||
h1{ font-size:18px;}
|
||||
img { display:block}
|
||||
'''
|
||||
'''#
|
||||
|
||||
def get_cover_url(self):
|
||||
soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html')
|
||||
@ -88,16 +77,18 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe):
|
||||
cov2 = str(cov)
|
||||
cov2=cov2[27:-18]
|
||||
#cov2 now is pic url, now go back to original function
|
||||
br = mechanize.Browser()
|
||||
br = browser()
|
||||
br.set_handle_redirect(False)
|
||||
try:
|
||||
br.open_novisit(cov2)
|
||||
cover_url = cov2
|
||||
except:
|
||||
cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg'
|
||||
cover_url ='http://profile.ak.fbcdn.net/hprofile-ak-snc4/373019_6149699161_1710984811_n.jpg'
|
||||
|
||||
# print '******** string is ', cov2,' ***'
|
||||
#cover_url = cov2
|
||||
#cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'
|
||||
return cover_url
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
import re, mechanize
|
||||
import re, random
|
||||
|
||||
from calibre import browser
|
||||
from calibre.web.feeds.recipes import BasicNewsRecipe
|
||||
|
||||
class AdvancedUserRecipe1325006965(BasicNewsRecipe):
|
||||
|
||||
title = u'The Sun UK'
|
||||
|
||||
description = 'A Recipe for The Sun tabloid UK'
|
||||
__author__ = 'Dave Asbury'
|
||||
# last updated 7/4/12
|
||||
# last updated 29/4/12
|
||||
language = 'en_GB'
|
||||
oldest_article = 1
|
||||
max_articles_per_feed = 15
|
||||
@ -48,12 +50,10 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
|
||||
|
||||
|
||||
feeds = [
|
||||
(u'News','http://feed43.com/2517447382644748.xml'),
|
||||
(u'Sport', u'http://feed43.com/4283846255668687.xml'),
|
||||
(u'Bizarre', u'http://feed43.com/0233840304242011.xml'),
|
||||
(u'Film',u'http://feed43.com/1307545221226200.xml'),
|
||||
(u'Music',u'http://feed43.com/1701513435064132.xml'),
|
||||
(u'Sun Woman',u'http://feed43.com/0022626854226453.xml'),
|
||||
(u'News', u'http://www.thesun.co.uk/sol/homepage/news/rss'),
|
||||
(u'Sport', u'http://www.thesun.co.uk/sol/homepage/sport/rss'),
|
||||
(u'Showbiz', u'http://www.thesun.co.uk/sol/homepage/showbiz/rss'),
|
||||
(u'Woman', u'http://www.thesun.co.uk/sol/homepage/woman/rss'),
|
||||
]
|
||||
|
||||
def get_cover_url(self):
|
||||
@ -61,14 +61,11 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
|
||||
# look for the block containing the sun button and url
|
||||
cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_84.gif);'})
|
||||
|
||||
|
||||
|
||||
#cov = soup.find(attrs={'id' : 'large'})
|
||||
cov2 = str(cov)
|
||||
|
||||
cov2='http://www.politicshome.com'+cov2[9:-133]
|
||||
#cov2 now contains url of the page containing pic
|
||||
|
||||
#cov2 now contains url of the page containing pic
|
||||
soup = self.index_to_soup(cov2)
|
||||
cov = soup.find(attrs={'id' : 'large'})
|
||||
@ -76,16 +73,21 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
|
||||
cov2=cov2[27:-18]
|
||||
#cov2 now is pic url, now go back to original function
|
||||
|
||||
br = mechanize.Browser()
|
||||
br = browser()
|
||||
br.set_handle_redirect(False)
|
||||
try:
|
||||
br.open_novisit(cov2)
|
||||
cover_url = cov2
|
||||
except:
|
||||
cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'
|
||||
cover_url = random.choice((
|
||||
'http://img.thesun.co.uk/multimedia/archive/00905/errorpage6_677961a_905507a.jpg'
|
||||
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage7_677962a_905505a.jpg'
|
||||
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage5_677960a_905512a.jpg'
|
||||
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage2_677957a_905502a.jpg'
|
||||
,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage3_677958a_905503a.jpg'
|
||||
))
|
||||
|
||||
#cover_url = cov2
|
||||
#cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'
|
||||
return cover_url
|
||||
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user