searxng/manage

#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later

# shellcheck disable=SC2034
main_cmd="$(basename "$0")"

# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"

# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"

# shellcheck source=utils/lib_sxng_container.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_container.sh"

# shellcheck source=utils/lib_sxng_data.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh"

# shellcheck source=utils/lib_sxng_weblate.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_weblate.sh"

# shellcheck source=utils/lib_sxng_static.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_static.sh"

# shellcheck source=utils/lib_sxng_node.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_node.sh"

# shellcheck source=utils/lib_sxng_themes.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"

# shellcheck source=utils/lib_sxng_test.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"

# shellcheck source=utils/lib_govm.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_govm.sh"

# shellcheck source=utils/lib_valkey.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"

# shellcheck source=utils/lib_sxng_vite.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh"

# add ./local dev tools from python (virtualenv), golang and nodejs
PATH="${PY_ENV}/bin:${REPO_ROOT}/node_modules/.bin:${GOROOT}/bin:${GOPATH}/bin:${PATH}"

# config

PYOBJECTS="searx"
PY_SETUP_EXTRAS='[test]'
GECKODRIVER_VERSION="v0.36.0"
# SPHINXOPTS=
BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")

YAMLLINT_FILES=()
while IFS= read -r line; do
    if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
        YAMLLINT_FILES+=("$line")
    fi
done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"

RST_FILES=(
    'README.rst'
)

SHFMT_SCRIPTS=(
    "./manage"
    "./container"
    "./utils"
)

help() {
    nvm.help
    cat <<EOF
webapp.:
  run       : run developer instance
docs.:
  html      : build HTML documentation
  live      : autobuild HTML documentation while editing
  gh-pages  : deploy on gh-pages branch
  prebuild  : build reST include files (./${DOCS_BUILD}/includes)
  clean     : clean documentation build
gecko.driver:
  download & install geckodriver if not already installed (required for
  robot_tests)
valkey:
  install   : create user (${VALKEY_USER}) and install systemd service (${VALKEY_SERVICE_NAME})
py.:
  build     : Build python packages at ./${PYDIST}
  clean     : delete virtualenv and intermediate py files
pyenv.:
  install   : developer install of SearXNG into virtualenv
  uninstall : uninstall developer installation
  cmd ...   : run command ... in virtualenv
  OK        : test if virtualenv is OK
format.:
  python    : format Python code source using black
  shell     : format Shell scripts using shfmt
EOF
    go.help
    node.help
    weblate.help
    container.help
    data.help
    test.help
    themes.help
    static.help
    vite.help
    cat <<EOF
dev.:
  env: enter developer environment (or exec a command in)
environment ...
  SEARXNG_VALKEY_URL : ${SEARXNG_VALKEY_URL}
EOF
}

dev.env() {
    go.env.dev
    nvm.env
    node.env.dev

    export GOENV

    if [ -z "$1" ]; then
        export PS1="(dev.env)$ "
        bash --norc --noprofile
    else
        "$@"
    fi
}

if [ "$VERBOSE" = "1" ]; then
    SPHINX_VERBOSE="-v"
    PYLINT_VERBOSE="-v"
fi

# needed by sphinx-docs
export DOCS_BUILD

webapp.run() {
    local parent_proc="$$"
    (
        if [ "${LIVE_THEME}" ]; then
            (themes.live "${LIVE_THEME}")
            kill $parent_proc
        fi
    ) &
    (
        sleep 3
        xdg-open http://127.0.0.1:8888/
    ) &
    SEARXNG_DEBUG=1 \
        GRANIAN_RELOAD="true" \
        GRANIAN_RELOAD_IGNORE_WORKER_FAILURE="true" \
        GRANIAN_RELOAD_PATHS="./searx" \
        GRANIAN_PROCESS_NAME="searxng" \
        GRANIAN_INTERFACE="wsgi" \
        GRANIAN_HOST="::" \
        GRANIAN_PORT="8888" \
        GRANIAN_WEBSOCKETS="false" \
        GRANIAN_LOOP="uvloop" \
        GRANIAN_BLOCKING_THREADS="4" \
        GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
        GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m" \
        pyenv.cmd granian searx.webapp:app
}

# shellcheck disable=SC2119
gecko.driver() {
    pyenv.install

    build_msg INSTALL "gecko.driver"
    # run installation in a subprocess and activate pyenv
    (
        set -e
        pyenv.activate

        INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
        set +e
        if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
            build_msg INSTALL "geckodriver already installed"
            return
        fi
        PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
        case "$PLATFORM" in
            "linux 32bit" | "linux2 32bit") ARCH="linux32" ;;
            "linux 64bit" | "linux2 64bit") ARCH="linux64" ;;
            "windows 32 bit") ARCH="win32" ;;
            "windows 64 bit") ARCH="win64" ;;
            "mac 64bit") ARCH="macos" ;;
        esac
        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"

        build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"

        FILE="$(mktemp)"
        wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
        rm -- "$FILE"
        chmod 755 -- "${PY_ENV_BIN}/geckodriver"
    )
    dump_return $?
}

py.build() {
    build_msg BUILD "python package ${PYDIST}"
    pyenv.cmd python setup.py \
        sdist -d "${PYDIST}" \
        bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
}

py.clean() {
    build_msg CLEAN pyenv
    (
        set -e
        pyenv.drop
        [ "$VERBOSE" = "1" ] && set -x
        rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
        find . -name '*.pyc' -exec rm -f {} +
        find . -name '*.pyo' -exec rm -f {} +
        find . -name __pycache__ -exec rm -rf {} +
    )
}

pyenv.check() {
    cat <<EOF
import yaml
print('import yaml --> OK')
EOF
}

pyenv.install() {

    if ! pyenv.OK; then
        py.clean >/dev/null
    fi
    if pyenv.install.OK >/dev/null; then
        return 0
    fi

    (
        set -e
        pyenv
        build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'"
        "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}"
    )
    local exit_val=$?
    if [ ! $exit_val -eq 0 ]; then
        die 42 "error while pip install (${PY_ENV_BIN})"
    fi
}

pyenv.uninstall() {
    build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
    pyenv.cmd python setup.py develop --uninstall 2>&1 |
        prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "

}

format.python() {
    build_msg TEST "[format.python] black ${BLACK_TARGETS[*]}"
    pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
    dump_return $?
}

format.shell() {
    build_msg TEST "[shfmt] shfmt ${SHFMT_SCRIPTS[*]}"
    go.tool shfmt --list --write "${SHFMT_SCRIPTS[@]}"
    dump_return $?
}

docs.prebuild() {
    build_msg DOCS "build ${DOCS_BUILD}/includes"
    (
        set -e
        [ "$VERBOSE" = "1" ] && set -x
        mkdir -p "${DOCS_BUILD}/includes"
        ./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst"
        pyenv.cmd searxng_extra/docs_prebuild
    )
    dump_return $?
}

# shellcheck disable=SC2119
main() {

    local _type
    local cmd="$1"
    shift

    if [ "$cmd" == "" ]; then
        help
        err_msg "missing command"
        return 42
    fi

    case "$cmd" in
        --getenv)
            var="$1"
            echo "${!var}"
            ;;
        --help) help ;;
        --*)
            help
            err_msg "unknown option $cmd"
            return 42
            ;;
        *)
            _type="$(type -t "$cmd")"
            if [ "$_type" != 'function' ]; then
                err_msg "unknown command: $cmd / use --help"
                return 42
            else
                "$cmd" "$@"
            fi
            ;;
    esac
}

main "$@"