[fix] engines - set hard timouts in *sub-request* (#5460)

The requests changed here all run outside of the network context timeout,
thereby preventing the engine's timeout from being applied (the engine's timeout
can become longer than it was configured).

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/archlinux.py

@@ -120,7 +120,7 @@ def fetch_traits(engine_traits: EngineTraits):
         'zh': 'Special:搜索',
     }
 
-    resp = get('https://wiki.archlinux.org/')
+    resp = get('https://wiki.archlinux.org/', timeout=3)
     if not resp.ok:  # type: ignore
         print("ERROR: response from wiki.archlinux.org is not OK.")
 

searx/engines/azure.py

@@ -95,7 +95,7 @@ def authenticate(t_id: str, c_id: str, c_secret: str) -> str:
         "scope": "https://management.azure.com/.default",
     }
 
-    resp: SXNG_Response = http_post(url, body)
+    resp: SXNG_Response = http_post(url, body, timeout=5)
     if resp.status_code != 200:
         raise RuntimeError(f"Azure authentication failed (status {resp.status_code}): {resp.text}")
     return resp.json()["access_token"]

searx/engines/presearch.py

@@ -140,7 +140,7 @@ def _get_request_id(query, params):
         if l.territory:
             headers['Accept-Language'] = f"{l.language}-{l.territory},{l.language};" "q=0.9,*;" "q=0.5"
 
-    resp = get(url, headers=headers)
+    resp = get(url, headers=headers, timout=5)
 
     for line in resp.text.split("\n"):
         if "window.searchId = " in line:

searx/engines/public_domain_image_archive.py

@@ -64,7 +64,7 @@ def _get_algolia_api_url():
         return __CACHED_API_URL
 
     # fake request to extract api url
-    resp = get(f"{pdia_base_url}/search/?q=")
+    resp = get(f"{pdia_base_url}/search/?q=", timeout=3)
     if resp.status_code != 200:
         raise LookupError("Failed to fetch config location (and as such the API url) for PDImageArchive")
     pdia_config_filepart = extr(resp.text, pdia_config_start, pdia_config_end)

searx/engines/pubmed.py

@@ -73,7 +73,7 @@ def request(query: str, params: "OnlineParams") -> None:
     )
     esearch_url = f"{eutils_api}/esearch.fcgi?{args}"
     # DTD: https://eutils.ncbi.nlm.nih.gov/eutils/dtd/20060628/esearch.dtd
-    esearch_resp: "SXNG_Response" = get(esearch_url)
+    esearch_resp: "SXNG_Response" = get(esearch_url, timeout=3)
     pmids_results = etree.XML(esearch_resp.content)
     pmids: list[str] = [i.text for i in pmids_results.xpath("//eSearchResult/IdList/Id")]
 

searx/engines/semantic_scholar.py

@@ -66,7 +66,7 @@ def setup(engine_settings: dict[str, t.Any]) -> bool:
 def get_ui_version() -> str:
     ret_val: str = CACHE.get("X-S2-UI-Version")
     if not ret_val:
-        resp = get(base_url)
+        resp = get(base_url, timeout=3)
         if not resp.ok:
             raise RuntimeError("Can't determine Semantic Scholar UI version")
 

searx/engines/seznam.py

@@ -27,7 +27,7 @@ base_url = 'https://search.seznam.cz/'
 
 
 def request(query, params):
-    response_index = get(base_url, headers=params['headers'], raise_for_httperror=True)
+    response_index = get(base_url, headers=params['headers'], raise_for_httperror=True, timout=3)
     dom = html.fromstring(response_index.text)
 
     url_params = {

searx/engines/soundcloud.py

@@ -124,7 +124,7 @@ def get_client_id() -> str | None:
 
     client_id = ""
     url = "https://soundcloud.com"
-    resp = http_get(url, timeout=10)
+    resp = http_get(url, timeout=3)
 
     if not resp.ok:
         logger.error("init: GET %s failed", url)