From 48214035de7980c683f2d7dd7b1e36a17050d3b9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 1 May 2012 09:17:20 +0530 Subject: [PATCH] Update Daily Mirrot and The Sun --- recipes/daily_mirror.recipe | 83 +++++++++++++++++-------------------- recipes/the_sun.recipe | 34 ++++++++------- 2 files changed, 55 insertions(+), 62 deletions(-) diff --git a/recipes/daily_mirror.recipe b/recipes/daily_mirror.recipe index 8bac57951c..ebced64283 100644 --- a/recipes/daily_mirror.recipe +++ b/recipes/daily_mirror.recipe @@ -1,13 +1,13 @@ - from calibre.web.feeds.news import BasicNewsRecipe +from claibre import browser import re -import mechanize + class AdvancedUserRecipe1306061239(BasicNewsRecipe): title = u'The Daily Mirror' - description = 'News as provide by The Daily Mirror -UK' + description = 'News as provided by The Daily Mirror -UK' __author__ = 'Dave Asbury' - # last updated 7/4/12 + # last updated 28/4/12 language = 'en_GB' #cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' @@ -15,89 +15,80 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe): oldest_article = 1 - max_articles_per_feed = 10 + max_articles_per_feed = 12 remove_empty_feeds = True remove_javascript = True no_stylesheets = True - auto_cleanup = True + # auto_cleanup = True #conversion_options = { 'linearize_tables' : True } - #keep_only_tags = [ - # dict(name='h1'), - # dict(name='div',attrs={'id' : 'body-content'}), - #dict(name='div',atts={'class' : 'article-body'}), + + keep_only_tags = [ dict(name='h1'), + dict(name='div',attrs={'class' : 'lead-text'}), + dict(name='div',attrs={'class' : 'styleGroup clearfix'}), + dict(name='div',attrs={'class' : 'widget relatedContents pictures widget-editable viziwyg-section-245 inpage-widget-158123'}), + dict(name='figure',attrs={'class' : 'clearfix'}), + dict(name='div',attrs={'class' :'body '}), + #dict(attrs={'class' : ['article-attr','byline append-1','published']}), #dict(name='p'), - # ] + ] - #remove_tags_after = [dict (name='div',attrs={'class' : 'related'})] remove_tags = [ + dict(attrs={'class' : 'comment'}), dict(name='title'), - dict(name='div',attrs={'class' : ['inline-ad span-16 last','caption']}), - # dict(name='div',attrs={'id' : ['sidebar','menu','search-box','roffers-top']}), - #dict(name='div',attrs={'class' :['inline-ad span-16 last','article-resize','related','list teasers']}), - #dict(attrs={'class' : ['channellink','article-tags','replace','append-html']}), - ] - - # preprocess_regexps = [ - #(re.compile(r'
.*?
', re.IGNORECASE | re.DOTALL), lambda match: '')] - preprocess_regexps = [ - (re.compile(r'- mirror.co.uk', re.IGNORECASE | re.DOTALL), lambda match: '')] + dict(name='ul',attrs={'class' : 'clearfix breadcrumbs '}), + dict(name='ul',attrs={'id' : 'login-201109171215'}), + dict(name='div',attrs={'class' : ['inline-ad span-16 last','caption']}),#'widget navigation breadcrumb widget-editable viziwyg-section-198 inpage-widget-80721 span-17','image-credit' + ] preprocess_regexps = [ - (re.compile(r'Advertisement >>', re.IGNORECASE | re.DOTALL), lambda match: '')] + (re.compile(r'- mirror.co.uk', re.IGNORECASE | re.DOTALL), lambda match: '')] - #preprocess_regexps = [ - #(re.compile(r'Sponsored Links', re.IGNORECASE | re.DOTALL), lambda match: '')] feeds = [ + (u'News',u'http://www.mirror.co.uk/news/rss.xml'), + (u'Sports',u'http://www.mirror.co.uk/sport/rss.xml'), + (u'3AM',u'http://www.mirror.co.uk/3am/rss.xml'), + (u'Lifestyle',u'http://www.mirror.co.uk/lifestyle/rss.xml') - (u'UK News', u'http://feed43.com/0287771688643868.xml') - ,(u'Tech News', u'http://feed43.com/2455520588350501.xml') - ,(u'Weird World','http://feed43.com/0863800333634654.xml') - ,(u'Sport','http://feed43.com/7713243036546130.xml') - ,(u'Sport : Boxing ','http://feed43.com/0414732220804255.xml') - ,(u'Sport : Rugby Union','http://feed43.com/4710138762362383.xml') - ,(u'Sport : Other','http://feed43.com/4501416886323415.xml') - ,(u'TV and Film','http://feed43.com/5238302853765104.xml') - ,(u'Celebs','http://feed43.com/8770061048844683.xml') - ,(u'Life Style : Family','http://feed43.com/4356170742410338.xml') - ,(u'Travel','http://feed43.com/1436576006476607.xml') # example of commented out feed not needed ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml') ] extra_css = ''' - body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} - h1{ font-size:18px;} - img { display:block} - ''' + h1{ font-size:medium;} + body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} + img { display:block} + '''# def get_cover_url(self): soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html') - # look for the block containing the mirror button and url +# look for the block containing the mirror button and url cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_92.gif);'}) cov2 = str(cov) cov2='http://www.politicshome.com'+cov2[9:-142] - #cov2 now contains url of the page containing pic +#cov2 now contains url of the page containing pic soup = self.index_to_soup(cov2) cov = soup.find(attrs={'id' : 'large'}) cov2 = str(cov) cov2=cov2[27:-18] #cov2 now is pic url, now go back to original function - br = mechanize.Browser() + br = browser() br.set_handle_redirect(False) try: - br.open_novisit(cov2) - cover_url = cov2 + br.open_novisit(cov2) + cover_url = cov2 except: - cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' + cover_url ='http://profile.ak.fbcdn.net/hprofile-ak-snc4/373019_6149699161_1710984811_n.jpg' + # print '******** string is ', cov2,' ***' #cover_url = cov2 #cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png' return cover_url + diff --git a/recipes/the_sun.recipe b/recipes/the_sun.recipe index db74e003a0..11500430ff 100644 --- a/recipes/the_sun.recipe +++ b/recipes/the_sun.recipe @@ -1,12 +1,14 @@ -import re, mechanize +import re, random + +from calibre import browser from calibre.web.feeds.recipes import BasicNewsRecipe + class AdvancedUserRecipe1325006965(BasicNewsRecipe): title = u'The Sun UK' - description = 'A Recipe for The Sun tabloid UK' __author__ = 'Dave Asbury' - # last updated 7/4/12 + # last updated 29/4/12 language = 'en_GB' oldest_article = 1 max_articles_per_feed = 15 @@ -48,12 +50,10 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe): feeds = [ - (u'News','http://feed43.com/2517447382644748.xml'), - (u'Sport', u'http://feed43.com/4283846255668687.xml'), - (u'Bizarre', u'http://feed43.com/0233840304242011.xml'), - (u'Film',u'http://feed43.com/1307545221226200.xml'), - (u'Music',u'http://feed43.com/1701513435064132.xml'), - (u'Sun Woman',u'http://feed43.com/0022626854226453.xml'), + (u'News', u'http://www.thesun.co.uk/sol/homepage/news/rss'), + (u'Sport', u'http://www.thesun.co.uk/sol/homepage/sport/rss'), + (u'Showbiz', u'http://www.thesun.co.uk/sol/homepage/showbiz/rss'), + (u'Woman', u'http://www.thesun.co.uk/sol/homepage/woman/rss'), ] def get_cover_url(self): @@ -61,14 +61,11 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe): # look for the block containing the sun button and url cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_84.gif);'}) - - #cov = soup.find(attrs={'id' : 'large'}) cov2 = str(cov) cov2='http://www.politicshome.com'+cov2[9:-133] #cov2 now contains url of the page containing pic - #cov2 now contains url of the page containing pic soup = self.index_to_soup(cov2) cov = soup.find(attrs={'id' : 'large'}) @@ -76,16 +73,21 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe): cov2=cov2[27:-18] #cov2 now is pic url, now go back to original function - br = mechanize.Browser() + br = browser() br.set_handle_redirect(False) try: br.open_novisit(cov2) cover_url = cov2 except: - cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png' + cover_url = random.choice(( + 'http://img.thesun.co.uk/multimedia/archive/00905/errorpage6_677961a_905507a.jpg' + ,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage7_677962a_905505a.jpg' + ,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage5_677960a_905512a.jpg' + ,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage2_677957a_905502a.jpg' + ,'http://img.thesun.co.uk/multimedia/archive/00905/errorpage3_677958a_905503a.jpg' + )) - #cover_url = cov2 - #cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png' return cover_url +