from calibre.web.feeds.news import BasicNewsRecipe from calibre import browser import re class AdvancedUserRecipe1306061239(BasicNewsRecipe): title = u'The Daily Mirror' description = 'News as provided by The Daily Mirror -UK' __author__ = 'Dave Asbury' # last updated 1/5/14 language = 'en_GB' # cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' masthead_url = 'http://www.nmauk.co.uk/nma/images/daily_mirror.gif' compress_news_images = True compress_news_images_max_size = 15 oldest_article = 1.5 max_articles_per_feed = 10 remove_empty_feeds = True remove_javascript = True no_stylesheets = True ignore_duplicate_articles = {'title'} keep_only_tags = [ dict(name='h1'), dict(name='div', attrs={'class': 'lead-text'}), dict(name='figure', attrs={'class': 'inline-image clearfix '}), dict(name='div', attrs={'class': 'styleGroup clearfix'}), dict(name='div', attrs={ 'class': 'widget relatedContents pictures widget-editable viziwyg-section-70 inpage-widget-2230659'}), dict(name='div', attrs={'class': 'body '}), ] remove_tags = [ dict(attrs={'class': [ 'article sa-teaser type-opinion', 'image-gallery', 'gallery-caption']}), dict(attrs={'class': 'comment'}), dict(name='title'), dict(name='ul'), dict(name='ul', attrs={'class': 'clearfix breadcrumbs '}), dict(name='ul', attrs={'id': 'login-201109171215'}), dict(name='div', attrs={ 'class': ['inline-ad span-16 last', 'caption']}), ] preprocess_regexps = [ (re.compile(r'- mirror.co.uk', re.IGNORECASE | re.DOTALL), lambda match: ''), (re.compile(r'Getty', re.IGNORECASE | re.DOTALL), lambda match: ''), ] feeds = [ (u'UK News', u'http://www.mirror.co.uk/news/uk-news/rss.xml'), (u'world News', u'http://www.mirror.co.uk/news/world-news/rss.xml'), (u'Sports', u'http://www.mirror.co.uk/sport/rss.xml'), (u'3AM', u'http://www.mirror.co.uk/3am/rss.xml'), (u'Lifestyle', u'http://www.mirror.co.uk/lifestyle/rss.xml') # example of commented out feed not needed # ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml') ] extra_css = ''' h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} p{font-family:Arial,Helvetica,sans-serif;font-size:small;} body{font-family:Helvetica,Arial,sans-serif;font-size:small;} ''' def get_cover_url(self): soup = self.index_to_soup( 'http://www.politicshome.com/uk/latest_frontpage.html') # look for the block containing the mirror button and url cov = soup.find(attrs={ 'style': 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_92.gif);'}) cov2 = str(cov) cov2 = 'http://www.politicshome.com' + cov2[9:-142] # cov2 now contains url of the page containing pic soup = self.index_to_soup(cov2) cov = soup.find(attrs={'id': 'large'}) cov = str(cov) cov2 = re.findall( r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', cov) cov2 = str(cov2) cov2 = cov2[2:len(cov2) - 2] # cov2 now is pic url, now go back to original function br = browser() br.set_handle_redirect(False) try: br.open_novisit(cov2) cover_url = cov2 except: cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/373019_6149699161_1710984811_n.jpg' return cover_url