mirror of
https://github.com/benbusby/whoogle-search.git
synced 2025-07-09 03:04:13 -04:00
Add UserAgent configuration (#1212)
* feat: Set Lynx as default user agent and improve custom user agent handling * chore: Remove user agent debug logging * Add User Agent wiki link, remove whitespace --------- Co-authored-by: Ben Busby <contact@benbusby.com>
This commit is contained in:
parent
041c1fbb35
commit
389c0a4d8d
@ -37,33 +37,12 @@ def get_rule_for_selector(stylesheet: CSSStyleSheet,
|
||||
|
||||
class Config:
|
||||
def __init__(self, **kwargs):
|
||||
app_config = current_app.config
|
||||
self.url = os.getenv('WHOOGLE_CONFIG_URL', '')
|
||||
self.lang_search = os.getenv('WHOOGLE_CONFIG_SEARCH_LANGUAGE', '')
|
||||
self.lang_interface = os.getenv('WHOOGLE_CONFIG_LANGUAGE', '')
|
||||
self.style_modified = os.getenv(
|
||||
'WHOOGLE_CONFIG_STYLE', '')
|
||||
self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
|
||||
self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
|
||||
self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
|
||||
self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', '')
|
||||
self.tbs = os.getenv('WHOOGLE_CONFIG_TIME_PERIOD', '')
|
||||
self.theme = os.getenv('WHOOGLE_CONFIG_THEME', 'system')
|
||||
self.safe = read_config_bool('WHOOGLE_CONFIG_SAFE')
|
||||
self.dark = read_config_bool('WHOOGLE_CONFIG_DARK') # deprecated
|
||||
self.alts = read_config_bool('WHOOGLE_CONFIG_ALTS')
|
||||
self.nojs = read_config_bool('WHOOGLE_CONFIG_NOJS')
|
||||
self.tor = read_config_bool('WHOOGLE_CONFIG_TOR')
|
||||
self.near = os.getenv('WHOOGLE_CONFIG_NEAR', '')
|
||||
self.new_tab = read_config_bool('WHOOGLE_CONFIG_NEW_TAB')
|
||||
self.view_image = read_config_bool('WHOOGLE_CONFIG_VIEW_IMAGE')
|
||||
self.get_only = read_config_bool('WHOOGLE_CONFIG_GET_ONLY')
|
||||
self.anon_view = read_config_bool('WHOOGLE_CONFIG_ANON_VIEW')
|
||||
self.preferences_encrypted = read_config_bool('WHOOGLE_CONFIG_PREFERENCES_ENCRYPTED')
|
||||
self.preferences_key = os.getenv('WHOOGLE_CONFIG_PREFERENCES_KEY', '')
|
||||
|
||||
self.accept_language = False
|
||||
# User agent configuration
|
||||
self.user_agent = kwargs.get('user_agent', 'LYNX_UA')
|
||||
self.custom_user_agent = kwargs.get('custom_user_agent', '')
|
||||
self.use_custom_user_agent = kwargs.get('use_custom_user_agent', False)
|
||||
|
||||
# Add user agent related keys to safe_keys
|
||||
self.safe_keys = [
|
||||
'lang_search',
|
||||
'lang_interface',
|
||||
@ -77,9 +56,39 @@ class Config:
|
||||
'nojs',
|
||||
'anon_view',
|
||||
'preferences_encrypted',
|
||||
'tbs'
|
||||
'tbs',
|
||||
'user_agent',
|
||||
'custom_user_agent',
|
||||
'use_custom_user_agent'
|
||||
]
|
||||
|
||||
app_config = current_app.config
|
||||
self.url = kwargs.get('url', '')
|
||||
self.lang_search = kwargs.get('lang_search', '')
|
||||
self.lang_interface = kwargs.get('lang_interface', '')
|
||||
self.style_modified = os.getenv(
|
||||
'WHOOGLE_CONFIG_STYLE', '')
|
||||
self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
|
||||
self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
|
||||
self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
|
||||
self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', '')
|
||||
self.tbs = os.getenv('WHOOGLE_CONFIG_TIME_PERIOD', '')
|
||||
self.theme = kwargs.get('theme', '')
|
||||
self.safe = kwargs.get('safe', '')
|
||||
self.dark = kwargs.get('dark', '')
|
||||
self.alts = kwargs.get('alts', '')
|
||||
self.nojs = kwargs.get('nojs', '')
|
||||
self.tor = kwargs.get('tor', '')
|
||||
self.near = kwargs.get('near', '')
|
||||
self.new_tab = kwargs.get('new_tab', '')
|
||||
self.view_image = kwargs.get('view_image', '')
|
||||
self.get_only = kwargs.get('get_only', '')
|
||||
self.anon_view = read_config_bool('WHOOGLE_CONFIG_ANON_VIEW')
|
||||
self.preferences_encrypted = read_config_bool('WHOOGLE_CONFIG_PREFERENCES_ENCRYPTED')
|
||||
self.preferences_key = os.getenv('WHOOGLE_CONFIG_PREFERENCES_KEY', '')
|
||||
|
||||
self.accept_language = False
|
||||
|
||||
# Skip setting custom config if there isn't one
|
||||
if kwargs:
|
||||
mutable_attrs = self.get_mutable_attrs()
|
||||
|
@ -18,7 +18,6 @@ AUTOCOMPLETE_URL = ('https://suggestqueries.google.com/'
|
||||
|
||||
MOBILE_UA = '{}/5.0 (Android 0; Mobile; rv:54.0) Gecko/54.0 {}/59.0'
|
||||
DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
|
||||
LYNX_UA = 'Lynx/2.9.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/3.4.0'
|
||||
|
||||
# Valid query params
|
||||
VALID_PARAMS = ['tbs', 'tbm', 'start', 'near', 'source', 'nfpr']
|
||||
@ -73,20 +72,19 @@ def send_tor_signal(signal: Signal) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def gen_user_agent(is_mobile) -> str:
|
||||
if True:
|
||||
# Temporary fix while the removal of javascript-free searches by
|
||||
# Google is being investigated
|
||||
def gen_user_agent(config, is_mobile) -> str:
|
||||
# Define the Lynx user agent
|
||||
LYNX_UA = 'Lynx/2.9.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/3.4.0'
|
||||
|
||||
# If using custom user agent, return the custom string
|
||||
if config.user_agent == 'custom' and config.custom_user_agent:
|
||||
return config.custom_user_agent
|
||||
|
||||
# If using Lynx user agent
|
||||
if config.user_agent == 'LYNX_UA':
|
||||
return LYNX_UA
|
||||
|
||||
user_agent = os.environ.get('WHOOGLE_USER_AGENT', '')
|
||||
user_agent_mobile = os.environ.get('WHOOGLE_USER_AGENT_MOBILE', '')
|
||||
if user_agent and not is_mobile:
|
||||
return user_agent
|
||||
|
||||
if user_agent_mobile and is_mobile:
|
||||
return user_agent_mobile
|
||||
|
||||
# If no custom user agent is set, generate a random one
|
||||
firefox = random.choice(['Choir', 'Squier', 'Higher', 'Wire']) + 'fox'
|
||||
linux = random.choice(['Win', 'Sin', 'Gin', 'Fin', 'Kin']) + 'ux'
|
||||
|
||||
@ -198,10 +196,7 @@ class Request:
|
||||
# enable Tor for future requests
|
||||
send_tor_signal(Signal.HEARTBEAT)
|
||||
|
||||
self.language = (
|
||||
config.lang_search if config.lang_search else ''
|
||||
)
|
||||
|
||||
self.language = config.lang_search if config.lang_search else ''
|
||||
self.country = config.country if config.country else ''
|
||||
|
||||
# For setting Accept-language Header
|
||||
@ -211,11 +206,13 @@ class Request:
|
||||
|
||||
self.mobile = bool(normal_ua) and ('Android' in normal_ua
|
||||
or 'iPhone' in normal_ua)
|
||||
self.modified_user_agent = gen_user_agent(self.mobile)
|
||||
if not self.mobile:
|
||||
self.modified_user_agent_mobile = gen_user_agent(True)
|
||||
|
||||
# Set up proxy, if previously configured
|
||||
# Generate user agent based on config
|
||||
self.modified_user_agent = gen_user_agent(config, self.mobile)
|
||||
if not self.mobile:
|
||||
self.modified_user_agent_mobile = gen_user_agent(config, True)
|
||||
|
||||
# Set up proxy configuration
|
||||
proxy_path = os.environ.get('WHOOGLE_PROXY_LOC', '')
|
||||
if proxy_path:
|
||||
proxy_type = os.environ.get('WHOOGLE_PROXY_TYPE', '')
|
||||
@ -235,6 +232,7 @@ class Request:
|
||||
'http': 'socks5://127.0.0.1:9050',
|
||||
'https': 'socks5://127.0.0.1:9050'
|
||||
} if config.tor else {}
|
||||
|
||||
self.tor = config.tor
|
||||
self.tor_valid = False
|
||||
self.root_path = root_path
|
||||
|
@ -438,8 +438,8 @@ def config():
|
||||
if name:
|
||||
config_pkl = os.path.join(app.config['CONFIG_PATH'], name)
|
||||
session['config'] = (pickle.load(open(config_pkl, 'rb'))
|
||||
if os.path.exists(config_pkl)
|
||||
else session['config'])
|
||||
if os.path.exists(config_pkl)
|
||||
else session['config'])
|
||||
return json.dumps(session['config'])
|
||||
else:
|
||||
return json.dumps({})
|
||||
@ -448,8 +448,20 @@ def config():
|
||||
if 'url' not in config_data or not config_data['url']:
|
||||
config_data['url'] = g.user_config.url
|
||||
|
||||
# Handle user agent configuration
|
||||
if 'user_agent' in config_data:
|
||||
if config_data['user_agent'] == 'custom':
|
||||
config_data['use_custom_user_agent'] = True
|
||||
# Keep both the selection and the custom string
|
||||
if 'custom_user_agent' in config_data:
|
||||
config_data['custom_user_agent'] = config_data['custom_user_agent']
|
||||
print(f"Setting custom user agent to: {config_data['custom_user_agent']}") # Debug log
|
||||
else:
|
||||
config_data['use_custom_user_agent'] = False
|
||||
config_data['custom_user_agent'] = ''
|
||||
|
||||
# Save config by name to allow a user to easily load later
|
||||
if 'name' in request.args:
|
||||
if name:
|
||||
pickle.dump(
|
||||
config_data,
|
||||
open(os.path.join(
|
||||
|
@ -228,6 +228,21 @@
|
||||
<input type="checkbox" name="get_only"
|
||||
id="config-get-only" {{ 'checked' if config.get_only else '' }}>
|
||||
</div>
|
||||
<div class="config-div config-div-user-agent">
|
||||
<label for="config-user-agent">User Agent: </label>
|
||||
<select name="user_agent" id="config-user-agent">
|
||||
<option value="LYNX_UA" {% if not config.user_agent or config.user_agent == 'LYNX_UA' %}selected{% endif %}>Lynx Browser</option>
|
||||
<option value="" {% if config.user_agent == '' and config.user_agent != 'LYNX_UA' %}selected{% endif %}>Original (Random)</option>
|
||||
<option value="custom" {% if config.user_agent == 'custom' %}selected{% endif %}>Custom</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="config-div config-div-custom-user-agent" {% if config.user_agent != 'custom' %}style="display: none;"{% endif %}>
|
||||
<label for="config-custom-user-agent">Custom User Agent: </label>
|
||||
<input type="text" name="custom_user_agent" id="config-custom-user-agent"
|
||||
value="{{ config.custom_user_agent }}"
|
||||
placeholder="Enter custom user agent string">
|
||||
<div><span class="info-text"> — <a href="https://github.com/benbusby/whoogle-search/wiki/User-Agents">User Agent Wiki</a></span></div>
|
||||
</div>
|
||||
<div class="config-div config-div-accept-language">
|
||||
<label for="config-accept-language">Set Accept-Language: </label>
|
||||
<input type="checkbox" name="accept_language"
|
||||
|
Loading…
x
Reference in New Issue
Block a user