Cutlery/whoogle-search
1
0
Fork 0
mirror of https://github.com/benbusby/whoogle-search.git synced 2025-07-09 03:04:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add UserAgent configuration (#1212)

Browse Source 
* feat: Set Lynx as default user agent and improve custom user agent handling

* chore: Remove user agent debug logging

* Add User Agent wiki link, remove whitespace

---------

Co-authored-by: Ben Busby <contact@benbusby.com>
This commit is contained in:
Don 2025-01-17 11:17:50 -06:00 committed by GitHub
parent 041c1fbb35
commit 389c0a4d8d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 85 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
app/models/config.py
View File

@ -37,33 +37,12 @@ def get_rule_for_selector(stylesheet: CSSStyleSheet,
class Config:
def __init__(self, **kwargs):
app_config = current_app.config
self.url = os.getenv('WHOOGLE_CONFIG_URL', '')
self.lang_search = os.getenv('WHOOGLE_CONFIG_SEARCH_LANGUAGE', '')
self.lang_interface = os.getenv('WHOOGLE_CONFIG_LANGUAGE', '')
self.style_modified = os.getenv(
'WHOOGLE_CONFIG_STYLE', '')
self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', '')
self.tbs = os.getenv('WHOOGLE_CONFIG_TIME_PERIOD', '')
self.theme = os.getenv('WHOOGLE_CONFIG_THEME', 'system')
self.safe = read_config_bool('WHOOGLE_CONFIG_SAFE')
self.dark = read_config_bool('WHOOGLE_CONFIG_DARK') # deprecated
self.alts = read_config_bool('WHOOGLE_CONFIG_ALTS')
self.nojs = read_config_bool('WHOOGLE_CONFIG_NOJS')
self.tor = read_config_bool('WHOOGLE_CONFIG_TOR')
self.near = os.getenv('WHOOGLE_CONFIG_NEAR', '')
self.new_tab = read_config_bool('WHOOGLE_CONFIG_NEW_TAB')
self.view_image = read_config_bool('WHOOGLE_CONFIG_VIEW_IMAGE')
self.get_only = read_config_bool('WHOOGLE_CONFIG_GET_ONLY')
self.anon_view = read_config_bool('WHOOGLE_CONFIG_ANON_VIEW')
self.preferences_encrypted = read_config_bool('WHOOGLE_CONFIG_PREFERENCES_ENCRYPTED')
self.preferences_key = os.getenv('WHOOGLE_CONFIG_PREFERENCES_KEY', '')
self.accept_language = False
# User agent configuration
self.user_agent = kwargs.get('user_agent', 'LYNX_UA')
self.custom_user_agent = kwargs.get('custom_user_agent', '')
self.use_custom_user_agent = kwargs.get('use_custom_user_agent', False)
# Add user agent related keys to safe_keys
self.safe_keys = [
'lang_search',
'lang_interface',
@ -77,9 +56,39 @@ class Config:
'nojs',
'anon_view',
'preferences_encrypted',
'tbs'
'tbs',
'user_agent',
'custom_user_agent',
'use_custom_user_agent'
]
app_config = current_app.config
self.url = kwargs.get('url', '')
self.lang_search = kwargs.get('lang_search', '')
self.lang_interface = kwargs.get('lang_interface', '')
self.style_modified = os.getenv(
'WHOOGLE_CONFIG_STYLE', '')
self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', '')
self.tbs = os.getenv('WHOOGLE_CONFIG_TIME_PERIOD', '')
self.theme = kwargs.get('theme', '')
self.safe = kwargs.get('safe', '')
self.dark = kwargs.get('dark', '')
self.alts = kwargs.get('alts', '')
self.nojs = kwargs.get('nojs', '')
self.tor = kwargs.get('tor', '')
self.near = kwargs.get('near', '')
self.new_tab = kwargs.get('new_tab', '')
self.view_image = kwargs.get('view_image', '')
self.get_only = kwargs.get('get_only', '')
self.anon_view = read_config_bool('WHOOGLE_CONFIG_ANON_VIEW')
self.preferences_encrypted = read_config_bool('WHOOGLE_CONFIG_PREFERENCES_ENCRYPTED')
self.preferences_key = os.getenv('WHOOGLE_CONFIG_PREFERENCES_KEY', '')
self.accept_language = False
# Skip setting custom config if there isn't one
if kwargs:
mutable_attrs = self.get_mutable_attrs()

40
app/request.py
View File

@ -18,7 +18,6 @@ AUTOCOMPLETE_URL = ('https://suggestqueries.google.com/'
MOBILE_UA = '{}/5.0 (Android 0; Mobile; rv:54.0) Gecko/54.0 {}/59.0'
DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
LYNX_UA = 'Lynx/2.9.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/3.4.0'
# Valid query params
VALID_PARAMS = ['tbs', 'tbm', 'start', 'near', 'source', 'nfpr']
@ -73,20 +72,19 @@ def send_tor_signal(signal: Signal) -> bool:
return False
def gen_user_agent(is_mobile) -> str:
if True:
# Temporary fix while the removal of javascript-free searches by
# Google is being investigated
def gen_user_agent(config, is_mobile) -> str:
# Define the Lynx user agent
LYNX_UA = 'Lynx/2.9.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/3.4.0'
# If using custom user agent, return the custom string
if config.user_agent == 'custom' and config.custom_user_agent:
return config.custom_user_agent
# If using Lynx user agent
if config.user_agent == 'LYNX_UA':
return LYNX_UA
user_agent = os.environ.get('WHOOGLE_USER_AGENT', '')
user_agent_mobile = os.environ.get('WHOOGLE_USER_AGENT_MOBILE', '')
if user_agent and not is_mobile:
return user_agent
if user_agent_mobile and is_mobile:
return user_agent_mobile
# If no custom user agent is set, generate a random one
firefox = random.choice(['Choir', 'Squier', 'Higher', 'Wire']) + 'fox'
linux = random.choice(['Win', 'Sin', 'Gin', 'Fin', 'Kin']) + 'ux'
@ -198,10 +196,7 @@ class Request:
# enable Tor for future requests
send_tor_signal(Signal.HEARTBEAT)
self.language = (
config.lang_search if config.lang_search else ''
)
self.language = config.lang_search if config.lang_search else ''
self.country = config.country if config.country else ''
# For setting Accept-language Header
@ -211,11 +206,13 @@ class Request:
self.mobile = bool(normal_ua) and ('Android' in normal_ua
or 'iPhone' in normal_ua)
self.modified_user_agent = gen_user_agent(self.mobile)
if not self.mobile:
self.modified_user_agent_mobile = gen_user_agent(True)
# Set up proxy, if previously configured
# Generate user agent based on config
self.modified_user_agent = gen_user_agent(config, self.mobile)
if not self.mobile:
self.modified_user_agent_mobile = gen_user_agent(config, True)
# Set up proxy configuration
proxy_path = os.environ.get('WHOOGLE_PROXY_LOC', '')
if proxy_path:
proxy_type = os.environ.get('WHOOGLE_PROXY_TYPE', '')
@ -235,6 +232,7 @@ class Request:
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'
} if config.tor else {}
self.tor = config.tor
self.tor_valid = False
self.root_path = root_path

18
app/routes.py
View File

@ -438,8 +438,8 @@ def config():
if name:
config_pkl = os.path.join(app.config['CONFIG_PATH'], name)
session['config'] = (pickle.load(open(config_pkl, 'rb'))
if os.path.exists(config_pkl)
else session['config'])
if os.path.exists(config_pkl)
else session['config'])
return json.dumps(session['config'])
else:
return json.dumps({})
@ -448,8 +448,20 @@ def config():
if 'url' not in config_data or not config_data['url']:
config_data['url'] = g.user_config.url
# Handle user agent configuration
if 'user_agent' in config_data:
if config_data['user_agent'] == 'custom':
config_data['use_custom_user_agent'] = True
# Keep both the selection and the custom string
if 'custom_user_agent' in config_data:
config_data['custom_user_agent'] = config_data['custom_user_agent']
print(f"Setting custom user agent to: {config_data['custom_user_agent']}") # Debug log
else:
config_data['use_custom_user_agent'] = False
config_data['custom_user_agent'] = ''
# Save config by name to allow a user to easily load later
if 'name' in request.args:
if name:
pickle.dump(
config_data,
open(os.path.join(

15
app/templates/index.html
View File

@ -228,6 +228,21 @@
<input type="checkbox" name="get_only"
id="config-get-only" {{ 'checked' if config.get_only else '' }}>
</div>
<div class="config-div config-div-user-agent">
<label for="config-user-agent">User Agent: </label>
<select name="user_agent" id="config-user-agent">
<option value="LYNX_UA" {% if not config.user_agent or config.user_agent == 'LYNX_UA' %}selected{% endif %}>Lynx Browser</option>
<option value="" {% if config.user_agent == '' and config.user_agent != 'LYNX_UA' %}selected{% endif %}>Original (Random)</option>
<option value="custom" {% if config.user_agent == 'custom' %}selected{% endif %}>Custom</option>
</select>
</div>
<div class="config-div config-div-custom-user-agent" {% if config.user_agent != 'custom' %}style="display: none;"{% endif %}>
<label for="config-custom-user-agent">Custom User Agent: </label>
<input type="text" name="custom_user_agent" id="config-custom-user-agent"
value="{{ config.custom_user_agent }}"
placeholder="Enter custom user agent string">
<div><span class="info-text"> — <a href="https://github.com/benbusby/whoogle-search/wiki/User-Agents">User Agent Wiki</a></span></div>
</div>
<div class="config-div config-div-accept-language">
<label for="config-accept-language">Set Accept-Language: </label>
<input type="checkbox" name="accept_language"