From 653cd41d50999f8f37c8518da26ddf37de1c6cc5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 17 Aug 2007 02:43:07 +0000 Subject: [PATCH] Use mechanize --- osx_installer.py | 2 +- src/libprs500/web/fetch/simple.py | 9 ++++----- windows_installer.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/osx_installer.py b/osx_installer.py index baaf6d5898..40d5fd91e6 100644 --- a/osx_installer.py +++ b/osx_installer.py @@ -233,7 +233,7 @@ setup( 'argv_emulation' : True, 'iconfile' : 'icons/library.icns', 'frameworks': ['libusb.dylib', 'libunrar.dylib'], - 'includes' : ['sip', 'pkg_resources', 'PyQt4.QtSvg'], + 'includes' : ['sip', 'pkg_resources', 'PyQt4.QtSvg', 'mechanize', 'ClientForm'], 'packages' : ['PIL', 'Authorization',], 'excludes' : ['pydoc'], 'plist' : { 'CFBundleGetInfoString' : '''libprs500, an E-book management application.''' diff --git a/src/libprs500/web/fetch/simple.py b/src/libprs500/web/fetch/simple.py index 2cf2548747..6938e42811 100644 --- a/src/libprs500/web/fetch/simple.py +++ b/src/libprs500/web/fetch/simple.py @@ -15,12 +15,12 @@ ''' Fetch a webpage and its links recursively. ''' -import sys, socket, urllib2, os, urlparse, codecs, logging, re, time, copy +import sys, socket, os, urlparse, codecs, logging, re, time, copy, urllib2 from urllib import url2pathname from httplib import responses from optparse import OptionParser -from libprs500 import __version__, __appname__, __author__, setup_cli_handlers +from libprs500 import __version__, __appname__, __author__, setup_cli_handlers, browser from libprs500.ebooks.BeautifulSoup import BeautifulSoup class FetchError(Exception): @@ -57,6 +57,7 @@ class RecursiveFetcher(object): os.makedirs(self.base_dir) self.default_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(options.timeout) + self.browser = options.browser if hasattr(options, 'browser') else browser() self.max_recursions = options.max_recursions self.match_regexps = [re.compile(i, re.IGNORECASE) for i in options.match_regexps] self.filter_regexps = [re.compile(i, re.IGNORECASE) for i in options.filter_regexps] @@ -84,9 +85,7 @@ class RecursiveFetcher(object): if delta < self.delay: time.sleep(delta) try: - opener = urllib2.build_opener() - opener.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; i686 Linux; en_US; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4')] - f = opener.open(url) + f = self.browser.open(url) except urllib2.URLError, err: if hasattr(err, 'code') and responses.has_key(err.code): raise FetchError, responses[err.code] diff --git a/windows_installer.py b/windows_installer.py index 8645721b5c..ff3c5c1d4c 100644 --- a/windows_installer.py +++ b/windows_installer.py @@ -433,7 +433,7 @@ setup( options = { 'py2exe' : {'compressed': 1, 'optimize' : 2, 'dist_dir' : PY2EXE_DIR, - 'includes' : ['sip', 'pkg_resources', 'PyQt4.QtSvg'], + 'includes' : ['sip', 'pkg_resources', 'PyQt4.QtSvg', 'mechanize', 'ClientForm'], 'packages' : ['PIL', 'WmfPlugin'], 'excludes' : ["Tkconstants", "Tkinter", "tcl", "_imagingtk", "ImageTk", "FixTk",