Add UserAgent configuration (#1212)

* feat: Set Lynx as default user agent and improve custom user agent handling

* chore: Remove user agent debug logging

* Add User Agent wiki link, remove whitespace

---------

Co-authored-by: Ben Busby <contact@benbusby.com>

app/models/config.py

@@ -37,33 +37,12 @@ def get_rule_for_selector(stylesheet: CSSStyleSheet,
 
 class Config:
     def __init__(self, **kwargs):
-        app_config = current_app.config
+        # User agent configuration
-        self.url = os.getenv('WHOOGLE_CONFIG_URL', '')
+        self.user_agent = kwargs.get('user_agent', 'LYNX_UA')
-        self.lang_search = os.getenv('WHOOGLE_CONFIG_SEARCH_LANGUAGE', '')
+        self.custom_user_agent = kwargs.get('custom_user_agent', '')
-        self.lang_interface = os.getenv('WHOOGLE_CONFIG_LANGUAGE', '')
+        self.use_custom_user_agent = kwargs.get('use_custom_user_agent', False)
-        self.style_modified = os.getenv(
-            'WHOOGLE_CONFIG_STYLE', '')
-        self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
-        self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
-        self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
-        self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', '')
-        self.tbs = os.getenv('WHOOGLE_CONFIG_TIME_PERIOD', '')
-        self.theme = os.getenv('WHOOGLE_CONFIG_THEME', 'system')
-        self.safe = read_config_bool('WHOOGLE_CONFIG_SAFE')
-        self.dark = read_config_bool('WHOOGLE_CONFIG_DARK')  # deprecated
-        self.alts = read_config_bool('WHOOGLE_CONFIG_ALTS')
-        self.nojs = read_config_bool('WHOOGLE_CONFIG_NOJS')
-        self.tor = read_config_bool('WHOOGLE_CONFIG_TOR')
-        self.near = os.getenv('WHOOGLE_CONFIG_NEAR', '')
-        self.new_tab = read_config_bool('WHOOGLE_CONFIG_NEW_TAB')
-        self.view_image = read_config_bool('WHOOGLE_CONFIG_VIEW_IMAGE')
-        self.get_only = read_config_bool('WHOOGLE_CONFIG_GET_ONLY')
-        self.anon_view = read_config_bool('WHOOGLE_CONFIG_ANON_VIEW')
-        self.preferences_encrypted = read_config_bool('WHOOGLE_CONFIG_PREFERENCES_ENCRYPTED')
-        self.preferences_key = os.getenv('WHOOGLE_CONFIG_PREFERENCES_KEY', '')
-
-        self.accept_language = False
 
+        # Add user agent related keys to safe_keys
         self.safe_keys = [
             'lang_search',
             'lang_interface',
@@ -77,9 +56,39 @@ class Config:
             'nojs',
             'anon_view',
             'preferences_encrypted',
-            'tbs'
+            'tbs',
+            'user_agent',
+            'custom_user_agent',
+            'use_custom_user_agent'
         ]
 
+        app_config = current_app.config
+        self.url = kwargs.get('url', '')
+        self.lang_search = kwargs.get('lang_search', '')
+        self.lang_interface = kwargs.get('lang_interface', '')
+        self.style_modified = os.getenv(
+            'WHOOGLE_CONFIG_STYLE', '')
+        self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
+        self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
+        self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
+        self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', '')
+        self.tbs = os.getenv('WHOOGLE_CONFIG_TIME_PERIOD', '')
+        self.theme = kwargs.get('theme', '')
+        self.safe = kwargs.get('safe', '')
+        self.dark = kwargs.get('dark', '')
+        self.alts = kwargs.get('alts', '')
+        self.nojs = kwargs.get('nojs', '')
+        self.tor = kwargs.get('tor', '')
+        self.near = kwargs.get('near', '')
+        self.new_tab = kwargs.get('new_tab', '')
+        self.view_image = kwargs.get('view_image', '')
+        self.get_only = kwargs.get('get_only', '')
+        self.anon_view = read_config_bool('WHOOGLE_CONFIG_ANON_VIEW')
+        self.preferences_encrypted = read_config_bool('WHOOGLE_CONFIG_PREFERENCES_ENCRYPTED')
+        self.preferences_key = os.getenv('WHOOGLE_CONFIG_PREFERENCES_KEY', '')
+
+        self.accept_language = False
+
         # Skip setting custom config if there isn't one
         if kwargs:
             mutable_attrs = self.get_mutable_attrs()

app/request.py

@@ -18,7 +18,6 @@ AUTOCOMPLETE_URL = ('https://suggestqueries.google.com/'
 
 MOBILE_UA = '{}/5.0 (Android 0; Mobile; rv:54.0) Gecko/54.0 {}/59.0'
 DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
-LYNX_UA = 'Lynx/2.9.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/3.4.0'
 
 # Valid query params
 VALID_PARAMS = ['tbs', 'tbm', 'start', 'near', 'source', 'nfpr']
@@ -73,20 +72,19 @@ def send_tor_signal(signal: Signal) -> bool:
     return False
 
 
-def gen_user_agent(is_mobile) -> str:
+def gen_user_agent(config, is_mobile) -> str:
-    if True:
+    # Define the Lynx user agent
-        # Temporary fix while the removal of javascript-free searches by
+    LYNX_UA = 'Lynx/2.9.2 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/3.4.0'
-        # Google is being investigated
+
+    # If using custom user agent, return the custom string
+    if config.user_agent == 'custom' and config.custom_user_agent:
+        return config.custom_user_agent
+
+    # If using Lynx user agent
+    if config.user_agent == 'LYNX_UA':
         return LYNX_UA
 
-    user_agent = os.environ.get('WHOOGLE_USER_AGENT', '')
+    # If no custom user agent is set, generate a random one
-    user_agent_mobile = os.environ.get('WHOOGLE_USER_AGENT_MOBILE', '')
-    if user_agent and not is_mobile:
-        return user_agent
-
-    if user_agent_mobile and is_mobile:
-        return user_agent_mobile
-
     firefox = random.choice(['Choir', 'Squier', 'Higher', 'Wire']) + 'fox'
     linux = random.choice(['Win', 'Sin', 'Gin', 'Fin', 'Kin']) + 'ux'
 
@@ -198,10 +196,7 @@ class Request:
         # enable Tor for future requests
         send_tor_signal(Signal.HEARTBEAT)
 
-        self.language = (
+        self.language = config.lang_search if config.lang_search else ''
-            config.lang_search if config.lang_search else ''
-        )
-
         self.country = config.country if config.country else ''
 
         # For setting Accept-language Header
@@ -211,11 +206,13 @@ class Request:
 
         self.mobile = bool(normal_ua) and ('Android' in normal_ua
                                            or 'iPhone' in normal_ua)
-        self.modified_user_agent = gen_user_agent(self.mobile)
-        if not self.mobile:
-            self.modified_user_agent_mobile = gen_user_agent(True)
 
-        # Set up proxy, if previously configured
+        # Generate user agent based on config
+        self.modified_user_agent = gen_user_agent(config, self.mobile)
+        if not self.mobile:
+            self.modified_user_agent_mobile = gen_user_agent(config, True)
+
+        # Set up proxy configuration
         proxy_path = os.environ.get('WHOOGLE_PROXY_LOC', '')
         if proxy_path:
             proxy_type = os.environ.get('WHOOGLE_PROXY_TYPE', '')
@@ -235,6 +232,7 @@ class Request:
                 'http': 'socks5://127.0.0.1:9050',
                 'https': 'socks5://127.0.0.1:9050'
             } if config.tor else {}
+
         self.tor = config.tor
         self.tor_valid = False
         self.root_path = root_path

app/routes.py

@@ -438,8 +438,8 @@ def config():
         if name:
             config_pkl = os.path.join(app.config['CONFIG_PATH'], name)
             session['config'] = (pickle.load(open(config_pkl, 'rb'))
-                                 if os.path.exists(config_pkl)
+                                if os.path.exists(config_pkl)
-                                 else session['config'])
+                                else session['config'])
             return json.dumps(session['config'])
         else:
             return json.dumps({})
@@ -448,8 +448,20 @@ def config():
         if 'url' not in config_data or not config_data['url']:
             config_data['url'] = g.user_config.url
 
+        # Handle user agent configuration
+        if 'user_agent' in config_data:
+            if config_data['user_agent'] == 'custom':
+                config_data['use_custom_user_agent'] = True
+                # Keep both the selection and the custom string
+                if 'custom_user_agent' in config_data:
+                    config_data['custom_user_agent'] = config_data['custom_user_agent']
+                    print(f"Setting custom user agent to: {config_data['custom_user_agent']}")  # Debug log
+            else:
+                config_data['use_custom_user_agent'] = False
+                config_data['custom_user_agent'] = ''
+
         # Save config by name to allow a user to easily load later
-        if 'name' in request.args:
+        if name:
             pickle.dump(
                 config_data,
                 open(os.path.join(

app/templates/index.html

@@ -228,6 +228,21 @@
                             <input type="checkbox" name="get_only"
                                    id="config-get-only" {{ 'checked' if config.get_only else '' }}>
                         </div>
+                        <div class="config-div config-div-user-agent">
+                            <label for="config-user-agent">User Agent: </label>
+                            <select name="user_agent" id="config-user-agent">
+                                <option value="LYNX_UA" {% if not config.user_agent or config.user_agent == 'LYNX_UA' %}selected{% endif %}>Lynx Browser</option>
+                                <option value="" {% if config.user_agent == '' and config.user_agent != 'LYNX_UA' %}selected{% endif %}>Original (Random)</option>
+                                <option value="custom" {% if config.user_agent == 'custom' %}selected{% endif %}>Custom</option>
+                            </select>
+                        </div>
+                        <div class="config-div config-div-custom-user-agent" {% if config.user_agent != 'custom' %}style="display: none;"{% endif %}>
+                            <label for="config-custom-user-agent">Custom User Agent: </label>
+                            <input type="text" name="custom_user_agent" id="config-custom-user-agent"
+                                   value="{{ config.custom_user_agent }}"
+                                   placeholder="Enter custom user agent string">
+                            <div><span class="info-text"> — <a href="https://github.com/benbusby/whoogle-search/wiki/User-Agents">User Agent Wiki</a></span></div>
+                        </div>
                         <div class="config-div config-div-accept-language">
                             <label for="config-accept-language">Set Accept-Language: </label>
                             <input type="checkbox" name="accept_language"