News download: Handle feeds that have entries with empty ids

This commit is contained in:
Kovid Goyal 2013-02-10 17:00:15 +05:30
parent 6aa1b67d88
commit fa67ceaeb2

View File

@ -152,11 +152,13 @@ class Feed(object):
for item in articles: for item in articles:
if len(self.articles) >= max_articles_per_feed: if len(self.articles) >= max_articles_per_feed:
break break
id = item.get('id', 'internal id#'+str(self.id_counter)) self.id_counter += 1
id = item.get('id', None)
if not id:
id = 'internal id#%s'%self.id_counter
if id in self.added_articles: if id in self.added_articles:
return return
self.added_articles.append(id) self.added_articles.append(id)
self.id_counter += 1
published = time.gmtime(item.get('timestamp', time.time())) published = time.gmtime(item.get('timestamp', time.time()))
title = item.get('title', _('Untitled article')) title = item.get('title', _('Untitled article'))
link = item.get('url', None) link = item.get('url', None)
@ -176,13 +178,15 @@ class Feed(object):
def parse_article(self, item): def parse_article(self, item):
id = item.get('id', 'internal id#'+str(self.id_counter)) self.id_counter += 1
id = item.get('id', None)
if not id:
id = 'internal id#%s'%self.id_counter
if id in self.added_articles: if id in self.added_articles:
return return
published = item.get('date_parsed', time.gmtime()) published = item.get('date_parsed', time.gmtime())
if not published: if not published:
published = time.gmtime() published = time.gmtime()
self.id_counter += 1
self.added_articles.append(id) self.added_articles.append(id)
title = item.get('title', _('Untitled article')) title = item.get('title', _('Untitled article'))