diff --git a/recipes/berlin_policy_journal.recipe b/recipes/berlin_policy_journal.recipe new file mode 100644 index 0000000000..31101027b3 --- /dev/null +++ b/recipes/berlin_policy_journal.recipe @@ -0,0 +1,100 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +from __future__ import unicode_literals + +__license__ = 'GPL v3' +__copyright__ = '2016, Aimylios' + +''' +berlinpolicyjournal.com +''' + +import re, time +from calibre.web.feeds.news import BasicNewsRecipe + +class BerlinPolicyJournal(BasicNewsRecipe): + title = 'Berlin Policy Journal' + __author__ = 'Aimylios' + description = 'Articles from berlinpolicyjournal.com' + publisher = 'Deutsche Gesellschaft für Auswärtige Politik e.V.' + publication_type = 'magazine' + language = 'en_DE' + + oldest_article = 75 + max_articles_per_feed = 30 + simultaneous_downloads = 10 + no_stylesheets = True + remove_javascript = True + + conversion_options = {'smarten_punctuation' : True, + 'publisher' : publisher} + + # uncomment this to reduce file size + # compress_news_images = True + # compress_news_images_max_size = 16 + + INDEX = 'http://berlinpolicyjournal.com/' + FRONTPAGE = INDEX + 'page/' + cover_source = INDEX + masthead_url = INDEX + 'IP/wp-content/uploads/2015/04/logo_bpj_header.gif' + + keep_only_tags = [dict(name='article')] + + remove_tags = [ + dict(name='div', attrs={'class':['hidden', 'meta-count', 'meta-share']}), + dict(name='span', attrs={'class':'ava-auth'}), + dict(name='img', attrs={'alt':re.compile("_store_120px_width$")}), + dict(name='img', attrs={'alt':re.compile("^bpj_app_")}), + dict(name='img', attrs={'alt':re.compile("^BPJ-Montage_")}), + dict(name='footer'), + dict(name='br') + ] + + remove_attributes = ['sizes', 'width', 'height', 'align'] + + extra_css = 'h1 {font-size: 1.6em; text-align: left} \ + .entry-subtitle {font-style: italic; margin-bottom: 1em} \ + .wp-caption-text {font-size: 0.6em; margin-top: 0em}' + + def get_cover_url(self): + soup = self.index_to_soup(self.cover_source) + img_div = soup.find('div', id='text-2') + self.cover_url = img_div.find('img', src=True)['src'] + return self.cover_url + + def parse_index(self): + articles = {} + for i in range(1,5): + soup = self.index_to_soup(self.FRONTPAGE + str(i)) + for div in soup.findAll('div', attrs={'class':'post-box-big'}): + timestamp = time.strptime(div.find('time')['datetime'], '%Y-%m-%dT%H:%M:%S+00:00') + article_age = time.time() - time.mktime(timestamp) + if article_age <= self.oldest_article*24*3600: + category = self.tag_to_string(div.findAll('a', attrs={'rel':'category'})[-1]) + if category not in articles: + articles[category] = [] + article_title = self.tag_to_string(div.find('h3', attrs={'class':'entry-title'}).a) + article_url = div.find('h3', attrs={'class':'entry-title'}).a['href'] + article_date = unicode(time.strftime(' [%a, %d %b %H:%M]', timestamp)) + article_desc = self.tag_to_string(div.find('div', attrs={'class':'i-summary'}).p) + articles[category].append({'title':article_title, + 'url':article_url, + 'date':article_date, + 'description':article_desc}) + feeds = [] + for feed in articles: + if '/' in feed: + feeds.insert(0, (feed, articles[feed])) + else: + feeds.append((feed, articles[feed])) + return feeds + + def postprocess_html(self, soup, first_fetch): + # clean up formatting of author(s) and date + div = soup.find('div', {'class':'meta-info'}) + authors = '' + for entry in div.findAll('span', {'class':'entry-author'}): + authors = authors + entry.a.span.renderContents().strip() + ', ' + date = div.find('time').renderContents().strip() + div.replaceWith('
' + authors[:-2] + ' (' + date + ')
') + return soup