[mod] engines: Yahoo in different languages (#4826)

BTW fix issue reported in [1]

[1] https://github.com/searxng/searxng/pull/4814#issuecomment-2896948787

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
Co-authored-by: Bnyro <bnyro@tutanota.com>

searx/engines/yahoo.py

@@ -44,6 +44,29 @@ time_range_dict = {
     'month': ('1m', 'm'),
 }
 
+region2domain = {
+    "CO": "co.search.yahoo.com",  # Colombia
+    "TH": "th.search.yahoo.com",  # Thailand
+    "VE": "ve.search.yahoo.com",  # Venezuela
+    "CL": "cl.search.yahoo.com",  # Chile
+    "HK": "hk.search.yahoo.com",  # Hong Kong
+    "PE": "pe.search.yahoo.com",  # Peru
+    "CA": "ca.search.yahoo.com",  # Canada
+    "DE": "de.search.yahoo.com",  # Germany
+    "FR": "fr.search.yahoo.com",  # France
+    "TW": "tw.search.yahoo.com",  # Taiwan
+    "GB": "uk.search.yahoo.com",  # United Kingdom
+    "UK": "uk.search.yahoo.com",
+    "BR": "br.search.yahoo.com",  # Brazil
+    "IN": "in.search.yahoo.com",  # India
+    "ES": "espanol.search.yahoo.com",  # Espanol
+    "PH": "ph.search.yahoo.com",  # Philippines
+    "AR": "ar.search.yahoo.com",  # Argentina
+    "MX": "mx.search.yahoo.com",  # Mexico
+    "SG": "sg.search.yahoo.com",  # Singapore
+}
+"""Map regions to domain"""
+
 lang2domain = {
     'zh_chs': 'hk.search.yahoo.com',
     'zh_cht': 'tw.search.yahoo.com',
@@ -65,40 +88,40 @@ lang2domain = {
 
 yahoo_languages = {
     "all": "any",
-    "ar": "ar",
+    "ar": "ar",  # Arabic
-    "bg": "bg",
+    "bg": "bg",  # Bulgarian
-    "cs": "cs",
+    "cs": "cs",  # Czech
-    "da": "da",
+    "da": "da",  # Danish
-    "de": "de",
+    "de": "de",  # German
-    "el": "el",
+    "el": "el",  # Greek
-    "en": "en",
+    "en": "en",  # English
-    "es": "es",
+    "es": "es",  # Spanish
-    "et": "et",
+    "et": "et",  # Estonian
-    "fi": "fi",
+    "fi": "fi",  # Finnish
-    "fr": "fr",
+    "fr": "fr",  # French
-    "he": "he",
+    "he": "he",  # Hebrew
-    "hr": "hr",
+    "hr": "hr",  # Croatian
-    "hu": "hu",
+    "hu": "hu",  # Hungarian
-    "it": "it",
+    "it": "it",  # Italian
-    "ja": "ja",
+    "ja": "ja",  # Japanese
-    "ko": "ko",
+    "ko": "ko",  # Korean
-    "lt": "lt",
+    "lt": "lt",  # Lithuanian
-    "lv": "lv",
+    "lv": "lv",  # Latvian
-    "nl": "nl",
+    "nl": "nl",  # Dutch
-    "no": "no",
+    "no": "no",  # Norwegian
-    "pl": "pl",
+    "pl": "pl",  # Polish
-    "pt": "pt",
+    "pt": "pt",  # Portuguese
-    "ro": "ro",
+    "ro": "ro",  # Romanian
-    "ru": "ru",
+    "ru": "ru",  # Russian
-    "sk": "sk",
+    "sk": "sk",  # Slovak
-    "sl": "sl",
+    "sl": "sl",  # Slovenian
-    "sv": "sv",
+    "sv": "sv",  # Swedish
-    "th": "th",
+    "th": "th",  # Thai
-    "tr": "tr",
+    "tr": "tr",  # Turkish
-    "zh": "zh_chs",
+    "zh": "zh_chs",  # Chinese (Simplified)
     "zh_Hans": "zh_chs",
     'zh-CN': "zh_chs",
-    "zh_Hant": "zh_cht",
+    "zh_Hant": "zh_cht",  # Chinese (Traditional)
     "zh-HK": "zh_cht",
     'zh-TW': "zh_cht",
 }
@@ -107,7 +130,7 @@ yahoo_languages = {
 def request(query, params):
     """build request"""
 
-    lang = params["language"].split("-")[0]
+    lang, region = (params["language"].split("-") + [None])[:2]
     lang = yahoo_languages.get(lang, "any")
 
     offset = (params['pageno'] - 1) * 7 + 1
@@ -127,9 +150,11 @@ def request(query, params):
         }
     )
 
-    domain = lang2domain.get(lang, '%s.search.yahoo.com' % lang)
+    domain = region2domain.get(region)
+    if not domain:
+        domain = lang2domain.get(lang, '%s.search.yahoo.com' % lang)
     params['url'] = 'https://%s/search?%s' % (domain, args)
-    return params
+    params['domain'] = domain
 
 
 def parse_url(url_string):
@@ -157,14 +182,22 @@ def response(resp):
     results = []
     dom = html.fromstring(resp.text)
 
+    url_xpath = './/div[contains(@class,"compTitle")]/h3/a/@href'
+    title_xpath = './/h3//a/@aria-label'
+
+    domain = resp.search_params['domain']
+    if domain == "search.yahoo.com":
+        url_xpath = './/div[contains(@class,"compTitle")]/a/@href'
+        title_xpath = './/div[contains(@class,"compTitle")]/a/h3/span'
+
     # parse results
     for result in eval_xpath_list(dom, '//div[contains(@class,"algo-sr")]'):
-        url = eval_xpath_getindex(result, './/div[contains(@class,"compTitle")]/a/@href', 0, default=None)
+        url = eval_xpath_getindex(result, url_xpath, 0, default=None)
         if url is None:
             continue
         url = parse_url(url)
 
-        title = eval_xpath_getindex(result, './/div[contains(@class,"compTitle")]/a/h3/span', 0, default='')
+        title = eval_xpath_getindex(result, title_xpath, 0, default='')
         title: str = extract_text(title)
         content = eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0, default='')
         content: str = extract_text(content, allow_none=True)