Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Get rid of robust_parser as we now depend on mechanize 0.3

Browse Source
This commit is contained in:
Kovid Goyal 2017-04-29 12:02:19 +05:30
parent 6421e06045
commit 8c591dfc68
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 3 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/calibre/__init__.py
View File

@ -405,7 +405,7 @@ def random_user_agent(choose=None, allow_ie=True):
return random.choice(ua_list) if choose is None else ua_list[choose]
def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None, use_robust_parser=False, verify_ssl_certificates=True):
def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None, verify_ssl_certificates=True, handle_refresh=True):
'''
Create a mechanize browser for web scraping. The browser handles cookies,
refresh requests and ignores robots.txt. Also uses proxy if available.
@ -415,12 +415,8 @@ def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None,
:param verify_ssl_certificates: If false SSL certificates errors are ignored
'''
from calibre.utils.browser import Browser
if use_robust_parser:
import mechanize
opener = Browser(factory=mechanize.RobustFactory(), verify_ssl=verify_ssl_certificates)
else:
opener = Browser(verify_ssl=verify_ssl_certificates)
opener.set_handle_refresh(True, max_time=max_time, honor_time=honor_time)
opener.set_handle_refresh(handle_refresh, max_time=max_time, honor_time=honor_time)
opener.set_handle_robots(False)
if user_agent is None:
user_agent = USER_AGENT_MOBILE if mobile_browser else USER_AGENT