Make user agent Mozilla/5.0

This commit is contained in:
Kovid Goyal 2007-07-20 01:21:13 +00:00
parent 8c3c93fca3
commit 5dd28389f9

View File

@ -85,7 +85,9 @@ class RecursiveFetcher(object):
if delta < self.delay: if delta < self.delay:
time.sleep(delta) time.sleep(delta)
try: try:
f = urllib2.urlopen(url) opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
f = opener.open(url)
except urllib2.URLError, err: except urllib2.URLError, err:
if hasattr(err, 'code') and responses.has_key(err.code): if hasattr(err, 'code') and responses.has_key(err.code):
raise FetchError, responses[err.code] raise FetchError, responses[err.code]