From 0adf19efaa9260a797b6b2ca13cecd60cbdcc79f Mon Sep 17 00:00:00 2001 From: aimylios Date: Sun, 10 Jul 2016 09:21:13 +0200 Subject: [PATCH] Update Berlin Policy Journal --- recipes/berlin_policy_journal.recipe | 81 +++++++++++++++------------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/recipes/berlin_policy_journal.recipe b/recipes/berlin_policy_journal.recipe index 9707628b32..1264aa9c01 100644 --- a/recipes/berlin_policy_journal.recipe +++ b/recipes/berlin_policy_journal.recipe @@ -1,9 +1,8 @@ #!/usr/bin/env python2 # vim:fileencoding=utf-8 -from __future__ import unicode_literals +# License: GPLv3 Copyright: 2016, Aimylios -__license__ = 'GPL v3' -__copyright__ = '2016, Aimylios' +from __future__ import unicode_literals, division, absolute_import, print_function ''' berlinpolicyjournal.com @@ -20,7 +19,7 @@ class BerlinPolicyJournal(BasicNewsRecipe): publication_type = 'magazine' language = 'en_DE' - oldest_article = 75 + oldest_article = 50 max_articles_per_feed = 30 simultaneous_downloads = 5 no_stylesheets = True @@ -29,14 +28,12 @@ class BerlinPolicyJournal(BasicNewsRecipe): conversion_options = {'smarten_punctuation' : True, 'publisher' : publisher} - # uncomment this to reduce file size - # compress_news_images = True - # compress_news_images_max_size = 16 - INDEX = 'http://berlinpolicyjournal.com/' masthead_url = INDEX + 'IP/wp-content/uploads/2015/04/logo_bpj_header.gif' - keep_only_tags = [dict(name='article')] + keep_only_tags = [ + dict(name='article') + ] remove_tags = [ dict(name='div', attrs={'class':['hidden', 'meta-count', 'meta-share']}), @@ -44,44 +41,52 @@ class BerlinPolicyJournal(BasicNewsRecipe): dict(name='img', attrs={'alt':re.compile('_store_120px_width$')}), dict(name='img', attrs={'alt':re.compile('^bpj_app_')}), dict(name='img', attrs={'alt':re.compile('^BPJ-Montage_')}), - dict(name='footer'), - dict(name='br') + dict(name=['link', 'footer', 'br']) ] remove_attributes = ['sizes', 'width', 'height', 'align'] extra_css = 'h1 {font-size: 1.6em; text-align: left} \ .entry-subtitle {font-style: italic; margin-bottom: 1em} \ + .wp-caption {margin-top: 1em} \ .wp-caption-text {font-size: 0.6em; margin-top: 0em}' def parse_index(self): - articles = {} - for i in range(1,5): - soup = self.index_to_soup(self.INDEX + 'page/' + str(i)) - if i == 1: - img_div = soup.find('div', {'id':'text-2'}) - self.cover_url = img_div.find('img', src=True)['src'] - for div in soup.findAll('div', {'class':'post-box-big'}): - timestamp = time.strptime(div.find('time')['datetime'], '%Y-%m-%dT%H:%M:%S+00:00') - article_age = time.time() - time.mktime(timestamp) - if article_age <= self.oldest_article*24*3600: - category = self.tag_to_string(div.findAll('a', {'rel':'category'})[-1]) - if category not in articles: - articles[category] = [] - article_title = self.tag_to_string(div.find('h3', {'class':'entry-title'}).a) - article_url = div.find('h3', {'class':'entry-title'}).a['href'] - article_date = unicode(time.strftime(' [%a, %d %b %H:%M]', timestamp)) - article_desc = self.tag_to_string(div.find('div', {'class':'i-summary'}).p) - articles[category].append({'title':article_title, - 'url':article_url, - 'date':article_date, - 'description':article_desc}) + soup = self.index_to_soup(self.INDEX) + img_div = soup.find('div', {'id':'text-2'}) + self.cover_url = img_div.find('img', src=True)['src'] + menu = soup.find('ul', {'id':re.compile('menu-ip')}) + submenus = menu.findAll('li', {'class':re.compile('item-has-children')}) + mag = submenus[0].find('li') + mag_name = self.tag_to_string(mag.a) + mag_url = mag.a['href'] + categories = [{'name':mag_name, 'url':mag_url, 'type':'magazine'}] + for blog in submenus[1].findAll('li'): + blog_name = self.tag_to_string(blog.a) + blog_url = blog.a['href'] + categories.append({'name':blog_name, 'url':blog_url, 'type':'blog'}) feeds = [] - for feed in articles: - if '/' in feed: - feeds.insert(0, (feed, articles[feed])) - else: - feeds.append((feed, articles[feed])) + for cat in categories: + cat['articles'] = [] + for i in ['1', '2']: + soup = self.index_to_soup(cat['url'] + '/page/' + i) + for div in soup.findAll('div', {'class':'post-box-big'}): + timestamp = time.strptime(div.find('time')['datetime'][:15], '%Y-%m-%dT%H:%M') + age = (time.time() - time.mktime(timestamp)) / (24 * 3600) + if age > self.oldest_article and cat['type'] == 'blog': + continue + article_title = self.tag_to_string(div.find('h3', {'class':'entry-title'}).a) + article_url = div.find('h3', {'class':'entry-title'}).a['href'] + article_date = unicode(time.strftime(' [%a, %d %b %H:%M]', timestamp)) + article_desc = self.tag_to_string(div.find('div', {'class':'i-summary'}).p) + cat['articles'].append({'title':article_title, + 'url':article_url, + 'date':article_date, + 'description':article_desc}) + if soup.find('div', {'class':'pagination'}) is None: + break + if cat['articles']: + feeds.append((cat['name'], cat['articles'])) return feeds def postprocess_html(self, soup, first_fetch): @@ -91,5 +96,5 @@ class BerlinPolicyJournal(BasicNewsRecipe): for entry in div.findAll('span', {'class':'entry-author'}): authors = authors + entry.a.span.renderContents().strip() + ', ' date = div.find('time').renderContents().strip() - div.replaceWith('
' + authors[:-2] + ' (' + date + ')
') + div.replaceWith('
' + date + ' | ' + authors[:-2] + '
') return soup