Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-06-23 15:30:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update Berlin Policy Journal

Browse Source
This commit is contained in:
Kovid Goyal 2016-04-24 22:18:13 +05:30
parent 3651c85635
commit aaf6b52f36
1 changed files with 13 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
recipes/berlin_policy_journal.recipe
View File

@ -22,7 +22,7 @@ class BerlinPolicyJournal(BasicNewsRecipe):
oldest_article = 75 oldest_article = 75
max_articles_per_feed = 30 max_articles_per_feed = 30
simultaneous_downloads = 10 simultaneous_downloads = 5
no_stylesheets = True no_stylesheets = True
remove_javascript = True remove_javascript = True
@ -34,8 +34,6 @@ class BerlinPolicyJournal(BasicNewsRecipe):
# compress_news_images_max_size = 16 # compress_news_images_max_size = 16
INDEX = 'http://berlinpolicyjournal.com/' INDEX = 'http://berlinpolicyjournal.com/'
FRONTPAGE = INDEX + 'page/'
cover_source = INDEX
masthead_url = INDEX + 'IP/wp-content/uploads/2015/04/logo_bpj_header.gif' masthead_url = INDEX + 'IP/wp-content/uploads/2015/04/logo_bpj_header.gif'
keep_only_tags = [dict(name='article')] keep_only_tags = [dict(name='article')]
@ -43,9 +41,9 @@ class BerlinPolicyJournal(BasicNewsRecipe):
remove_tags = [ remove_tags = [
dict(name='div', attrs={'class':['hidden', 'meta-count', 'meta-share']}), dict(name='div', attrs={'class':['hidden', 'meta-count', 'meta-share']}),
dict(name='span', attrs={'class':'ava-auth'}), dict(name='span', attrs={'class':'ava-auth'}),
dict(name='img', attrs={'alt':re.compile("_store_120px_width$")}), dict(name='img', attrs={'alt':re.compile('_store_120px_width$')}),
dict(name='img', attrs={'alt':re.compile("^bpj_app_")}), dict(name='img', attrs={'alt':re.compile('^bpj_app_')}),
dict(name='img', attrs={'alt':re.compile("^BPJ-Montage_")}), dict(name='img', attrs={'alt':re.compile('^BPJ-Montage_')}),
dict(name='footer'), dict(name='footer'),
dict(name='br') dict(name='br')
] ]
@ -56,27 +54,24 @@ class BerlinPolicyJournal(BasicNewsRecipe):
.entry-subtitle {font-style: italic; margin-bottom: 1em} \ .entry-subtitle {font-style: italic; margin-bottom: 1em} \
.wp-caption-text {font-size: 0.6em; margin-top: 0em}' .wp-caption-text {font-size: 0.6em; margin-top: 0em}'
def get_cover_url(self):
soup = self.index_to_soup(self.cover_source)
img_div = soup.find('div', id='text-2')
self.cover_url = img_div.find('img', src=True)['src']
return self.cover_url
def parse_index(self): def parse_index(self):
articles = {} articles = {}
for i in range(1,5): for i in range(1,5):
soup = self.index_to_soup(self.FRONTPAGE + str(i)) soup = self.index_to_soup(self.INDEX + 'page/' + str(i))
for div in soup.findAll('div', attrs={'class':'post-box-big'}): if i == 1:
img_div = soup.find('div', {'id':'text-2'})
self.cover_url = img_div.find('img', src=True)['src']
for div in soup.findAll('div', {'class':'post-box-big'}):
timestamp = time.strptime(div.find('time')['datetime'], '%Y-%m-%dT%H:%M:%S+00:00') timestamp = time.strptime(div.find('time')['datetime'], '%Y-%m-%dT%H:%M:%S+00:00')
article_age = time.time() - time.mktime(timestamp) article_age = time.time() - time.mktime(timestamp)
if article_age <= self.oldest_article*24*3600: if article_age <= self.oldest_article*24*3600:
category = self.tag_to_string(div.findAll('a', attrs={'rel':'category'})[-1]) category = self.tag_to_string(div.findAll('a', {'rel':'category'})[-1])
if category not in articles: if category not in articles:
articles[category] = [] articles[category] = []
article_title = self.tag_to_string(div.find('h3', attrs={'class':'entry-title'}).a) article_title = self.tag_to_string(div.find('h3', {'class':'entry-title'}).a)
article_url = div.find('h3', attrs={'class':'entry-title'}).a['href'] article_url = div.find('h3', {'class':'entry-title'}).a['href']
article_date = unicode(time.strftime(' [%a, %d %b %H:%M]', timestamp)) article_date = unicode(time.strftime(' [%a, %d %b %H:%M]', timestamp))
article_desc = self.tag_to_string(div.find('div', attrs={'class':'i-summary'}).p) article_desc = self.tag_to_string(div.find('div', {'class':'i-summary'}).p)
articles[category].append({'title':article_title, articles[category].append({'title':article_title,
'url':article_url, 'url':article_url,
'date':article_date, 'date':article_date,