From c2ef186d9fe9cfb0584ad4808b6d5631029d8ebc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 12 Apr 2016 06:45:44 +0530 Subject: [PATCH] Update Handelsblatt --- recipes/handelsblatt.recipe | 103 ++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 39 deletions(-) diff --git a/recipes/handelsblatt.recipe b/recipes/handelsblatt.recipe index 1be19d89b5..9389dbda2d 100644 --- a/recipes/handelsblatt.recipe +++ b/recipes/handelsblatt.recipe @@ -1,4 +1,6 @@ #!/usr/bin/env python2 +# vim:fileencoding=utf-8 +from __future__ import unicode_literals __license__ = 'GPL v3' __copyright__ = '2016, Aimylios' @@ -11,15 +13,15 @@ import re from calibre.web.feeds.news import BasicNewsRecipe class Handelsblatt(BasicNewsRecipe): - title = u'Handelsblatt' - __author__ = 'Aimylios' # based on the work of malfi and Hegi - description = u'RSS-Feeds von Handelsblatt.com' - publisher = 'Verlagsgruppe Handelsblatt GmbH' - category = 'news, politics, business, economy, Germany' - publication_type = 'newspaper' - language = 'de' + title = 'Handelsblatt' + __author__ = 'Aimylios' # based on the work of malfi and Hegi + description = 'RSS-Feeds von Handelsblatt.com' + publisher = 'Verlagsgruppe Handelsblatt GmbH' + publication_type = 'newspaper' + needs_subscription = 'optional' + language = 'de' + encoding = 'utf-8' - encoding = 'utf8' oldest_article = 4 max_articles_per_feed = 30 simultaneous_downloads = 20 @@ -32,57 +34,66 @@ class Handelsblatt(BasicNewsRecipe): 'publisher' : publisher} # uncomment this to reduce file size - # compress_news_images = True + # compress_news_images = True + # compress_news_images_max_size = 16 cover_source = 'https://kaufhaus.handelsblatt.com/downloads/handelsblatt-epaper-p1951.html' masthead_url = 'http://www.handelsblatt.com/images/logo_handelsblatt/11002806/7-formatOriginal.png' feeds = [ - (u'Top-Themen', u'http://www.handelsblatt.com/contentexport/feed/top-themen'), - (u'Politik', u'http://www.handelsblatt.com/contentexport/feed/politik'), - (u'Unternehmen', u'http://www.handelsblatt.com/contentexport/feed/unternehmen'), - (u'Finanzen', u'http://www.handelsblatt.com/contentexport/feed/finanzen'), - (u'Technologie', u'http://www.handelsblatt.com/contentexport/feed/technologie'), - (u'Panorama', u'http://www.handelsblatt.com/contentexport/feed/panorama'), - (u'Sport', u'http://www.handelsblatt.com/contentexport/feed/sport') - ] + ('Top-Themen', 'http://www.handelsblatt.com/contentexport/feed/top-themen'), + ('Politik', 'http://www.handelsblatt.com/contentexport/feed/politik'), + ('Unternehmen', 'http://www.handelsblatt.com/contentexport/feed/unternehmen'), + ('Finanzen', 'http://www.handelsblatt.com/contentexport/feed/finanzen'), + ('Technologie', 'http://www.handelsblatt.com/contentexport/feed/technologie'), + ('Panorama', 'http://www.handelsblatt.com/contentexport/feed/panorama'), + ('Sport', 'http://www.handelsblatt.com/contentexport/feed/sport') + ] keep_only_tags = [dict(name='div', attrs={'class':['vhb-article-container']})] remove_tags = [ - dict(name='span', attrs={'class':['vhb-media', 'vhb-colon']}), - dict(name='small', attrs={'class':['vhb-credit']}), - dict(name='aside', attrs={'class':['vhb-article-element vhb-left', - 'vhb-article-element vhb-left vhb-teasergallery', - 'vhb-article-element vhb-left vhb-shorttexts']}), - dict(name='article', attrs={'class':['vhb-imagegallery vhb-teaser', 'vhb-teaser vhb-type-video']}), - dict(name='div', attrs={'class':['fb-post']}), - dict(name='blockquote', attrs={'class':['twitter-tweet']}), - dict(name='a', attrs={'class':['twitter-follow-button']}) - ] + dict(name='span', attrs={'class':['vhb-colon', 'vhb-label-premium']}), + dict(name='aside', attrs={'class':['vhb-article-element vhb-left', + 'vhb-article-element vhb-left vhb-teasergallery', + 'vhb-article-element vhb-left vhb-shorttexts']}), + dict(name='article', attrs={'class':['vhb-imagegallery vhb-teaser', + 'vhb-teaser vhb-type-video']}), + dict(name='small', attrs={'class':['vhb-credit']}), + dict(name='div', attrs={'class':['white_content', 'fb-post']}), + dict(name='a', attrs={'class':['twitter-follow-button']}), + dict(name='blockquote') + ] preprocess_regexps = [ - # Insert ". " after "Place" in Place - (re.compile(r'([^<]+)()', - re.DOTALL|re.IGNORECASE), lambda match: match.group(1) + '. ' + match.group(2)), - # Insert ": " after "Title" in Title - (re.compile(r'([^<]+)()', - re.DOTALL|re.IGNORECASE), lambda match: match.group(1) + ': ' + match.group(2)) - ] + # Insert ". " after "Place" in Place + (re.compile(r'([^<]+)()', + re.DOTALL|re.IGNORECASE), lambda match: match.group(1) + '. ' + match.group(2)), + # Insert ": " after "Title" in Title + (re.compile(r'([^<]+)()', + re.DOTALL|re.IGNORECASE), lambda match: match.group(1) + ': ' + match.group(2)) + ] extra_css = 'h2 {text-align: left} \ h3 {font-size: 1em; text-align: left} \ h4 {font-size: 1em; text-align: left; margin-bottom: 0em} \ em {font-style: normal; font-weight: bold} \ .vhb-subline {font-size: 0.6em; text-transform: uppercase} \ - .vhb-article-caption {float: left; padding-right: 0.2em} \ - .vhb-article-author-cell ul {list-style-type: none; margin: 0em} \ .vhb-teaser-head {margin-top: 1em; margin-bottom: 1em} \ .vhb-caption-wrapper {font-size: 0.6em} \ .hcf-location-mark {font-weight: bold} \ - .panel-link {color: black; text-decoration: none} \ .panel-body p {margin-top: 0em}' + def get_browser(self): + br = BasicNewsRecipe.get_browser(self) + if self.username is not None and self.password is not None: + br.open('https://profil.vhb.de/sso/login?service=http://www.handelsblatt.com') + br.select_form(nr=0) + br['username'] = self.username + br['password'] = self.password + br.submit() + return br + def get_cover_url(self): soup = self.index_to_soup(self.cover_source) style = soup.find('img', alt='Handelsblatt ePaper', style=True)['style'] @@ -99,13 +110,27 @@ class Handelsblatt(BasicNewsRecipe): if article_container is None: self.abort_article() else: + # remove all local hyperlinks + for a in soup.findAll('a', {'href':True}): + if a['href'] and a['href'][0] in ['/', '#']: + a.replaceWith(a.renderContents()) return soup def postprocess_html(self, soup, first_fetch): + # convert lists of author(s) and date(s) into simple text + for cap in soup.findAll('div', {'class':re.compile('.*vhb-article-caption')}): + cap.replaceWith(cap.renderContents()) + for row in soup.findAll('div', {'class':'vhb-article-author-row'}): + for ul in row.findAll('ul'): + entry = '' + for li in ul.findAll(lambda tag: tag.name == 'li' and not tag.attrs): + entry = entry + li.renderContents() + ', ' + for li in ul.findAll(lambda tag: tag.name == 'li' and tag.attrs): + entry = entry + li.renderContents() + '
' + ul.parent.replaceWith(entry) # make sure that all figure captions (including the source) are shown # without linebreaks by using the alternative text given within # instead of the original text (which is oddly formatted) - article_figures = soup.findAll('figure', {'class':'vhb-image'}) - for fig in article_figures: + for fig in soup.findAll('figure', {'class':'vhb-image'}): fig.find('div', {'class':'vhb-caption'}).replaceWith(fig.find('img')['alt']) return soup