searxng/utils/fetch_languages.py

# -*- coding: utf-8 -*-

# This script generates languages.py from intersecting each engine's supported languages.
#
# Output files (engines_languages.json and languages.py)
# are written in current directory to avoid overwriting in case something goes wrong.

import json
import io
from sys import path
from babel import Locale, UnknownLocaleError
from babel.languages import get_global

path.append('../searx')  # noqa
from searx import settings
from searx.engines import initialize_engines, engines

# Output files.
engines_languages_file = 'engines_languages.json'
languages_file = 'languages.py'


# Fetchs supported languages for each engine and writes json file with those.
def fetch_supported_languages():

    engines_languages = {}
    names = list(engines)
    names.sort()

    for engine_name in names:
        if hasattr(engines[engine_name], 'fetch_supported_languages'):
            engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
            print("fetched %s languages from engine %s" % (
                len(engines_languages[engine_name]), engine_name))
            if type(engines_languages[engine_name]) == list:
                engines_languages[engine_name] = sorted(engines_languages[engine_name])

    # write json file
    with open(engines_languages_file, 'w', encoding='utf-8') as f:
        json.dump(engines_languages, f, indent=2, sort_keys=True)

    return engines_languages


# Get babel Locale object from lang_code if possible.
def get_locale(lang_code):
    try:
        locale = Locale.parse(lang_code, sep='-')
        return locale
    except (UnknownLocaleError, ValueError):
        return None


# Append engine_name to list of engines that support locale.
def add_engine_counter(lang_code, engine_name, languages):
    if lang_code in languages:
        if 'counter' not in languages[lang_code]:
            languages[lang_code]['counter'] = [engine_name]
        elif engine_name not in languages[lang_code]['counter']:
            languages[lang_code]['counter'].append(engine_name)


# Join all language lists.
# TODO: Add language names from engine's language list if name not known by babel.
def join_language_lists(engines_languages):
    language_list = {}
    for engine_name in engines_languages:
        for lang_code in engines_languages[engine_name]:

            # apply custom fixes if necessary
            if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
                lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
                                 if lang_code == alias)

            locale = get_locale(lang_code)

            # ensure that lang_code uses standard language and country codes
            if locale and locale.territory:
                lang_code = locale.language + '-' + locale.territory

            # add locale if it's not in list
            if lang_code not in language_list:
                if locale:
                    language_list[lang_code] = {'name': locale.get_language_name().title(),
                                                'english_name': locale.english_name,
                                                'country': locale.get_territory_name() or ''}

                    # also add language without country
                    if locale.language not in language_list:
                        language_list[locale.language] = {'name': locale.get_language_name().title(),
                                                          'english_name': locale.english_name}
                else:
                    language_list[lang_code] = {}

            # count engine for both language_country combination and language alone
            add_engine_counter(lang_code, engine_name, language_list)
            add_engine_counter(lang_code.split('-')[0], engine_name, language_list)

    return language_list


# Filter language list so it only includes the most supported languages and countries.
def filter_language_list(all_languages):
    min_supported_engines = 10
    main_engines = [engine_name for engine_name in engines.keys()
                    if 'general' in engines[engine_name].categories and
                       engines[engine_name].supported_languages and
                       not engines[engine_name].disabled]

    # filter list to include only languages supported by most engines or all default general engines
    filtered_languages = {code: lang for code, lang
                          in all_languages.items()
                          if (len(lang.get('counter', [])) >= min_supported_engines or
                              all(main_engine in lang.get('counter', [])
                                  for main_engine in main_engines))}

    return filtered_languages


# Add country codes to languages without one and filter out language codes.
def assign_country_codes(filtered_languages, all_languages):
    sorted_languages = sorted(all_languages,
                              key=lambda lang: len(all_languages[lang].get('counter', [])),
                              reverse=True)
    previous_lang = None
    previous_code = None
    countries = 0
    for current_code in sorted(filtered_languages):
        current_lang = current_code.split('-')[0]

        # count country codes per language
        if current_lang == previous_lang:
            countries += 1

        else:
            if previous_lang is not None:
                # if language has no single country code
                if countries == 0:
                    # try to get country code with most supported engines
                    for l in sorted_languages:
                        l_parts = l.split('-')
                        if len(l_parts) == 2 and l_parts[0] == previous_lang:
                            filtered_languages[l] = all_languages[l]
                            filtered_languages[l]['country'] = ''
                            countries = 1
                            break

                    if countries == 0:
                        # get most likely country code from babel
                        subtags = get_global('likely_subtags').get(previous_lang)
                        if subtags:
                            subtag_parts = subtags.split('_')
                            new_code = subtag_parts[0] + '-' + subtag_parts[-1]
                            filtered_languages[new_code] = all_languages[previous_lang]
                            countries = 1

                if countries == 1:
                    # remove countryless version of language if there's only one country
                    del filtered_languages[previous_lang]
                    if previous_code in filtered_languages:
                        filtered_languages[previous_code]['country'] = ''

            countries = 0
            previous_lang = current_lang

        previous_code = current_code


# Write languages.py.
def write_languages_file(languages):
    new_file = open(languages_file, 'wb')
    file_content = '# -*- coding: utf-8 -*-\n'\
                   + '# list of language codes\n'\
                   + '# this file is generated automatically by utils/update_search_languages.py\n'\
                   + '\nlanguage_codes = ('
    for code in sorted(languages):
        file_content += '\n    (u"' + code + '"'\
                        + ', u"' + languages[code]['name'].split(' (')[0] + '"'\
                        + ', u"' + languages[code].get('country', '') + '"'\
                        + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),'
    # remove last comma
    file_content = file_content[:-1]
    file_content += '\n)\n'
    new_file.write(file_content.encode('utf8'))
    new_file.close()


if __name__ == "__main__":
    initialize_engines(settings['engines'])
    engines_languages = fetch_supported_languages()
    all_languages = join_language_lists(engines_languages)
    filtered_languages = filter_language_list(all_languages)
    assign_country_codes(filtered_languages, all_languages)
    write_languages_file(filtered_languages)