[mod] add Golang ecosystem to the SearXNG toolchain

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

.gitignore

@@ -9,6 +9,7 @@ geckodriver.log
 .coverage
 coverage/
 
+.govm/
 .nvm/
 cache/
 build/

Makefile

@@ -32,7 +32,7 @@ install uninstall:
 	$(Q)./manage pyenv.$@
 
 PHONY += clean
-clean: py.clean docs.clean node.clean nvm.clean test.clean
+clean: py.clean docs.clean node.clean nvm.clean go.clean test.clean
 	$(Q)./manage build_msg CLEAN  "common files"
 	$(Q)find . -name '*.orig' -exec rm -f {} +
 	$(Q)find . -name '*.rej' -exec rm -f {} +
@@ -57,7 +57,7 @@ test.shell:
 		$(MTOOLS) \
 		utils/lib.sh \
 		utils/lib_sxng*.sh \
-		utils/lib_go.sh \
+		utils/lib_govm.sh \
 		utils/lib_nvm.sh \
 		utils/lib_redis.sh \
 		utils/lib_valkey.sh \
@@ -82,6 +82,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover
 MANAGE += themes.all themes.simple themes.fix themes.lint themes.test
 MANAGE += static.build.commit static.build.drop static.build.restore
 MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
+MANAGE += go.env.dev go.clean
 
 PHONY += $(MANAGE)
 

manage

@@ -32,8 +32,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"
 # shellcheck source=utils/lib_sxng_test.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"
 
-# shellcheck source=utils/lib_go.sh
+# shellcheck source=utils/lib_govm.sh
-source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_govm.sh"
 
 # shellcheck source=utils/lib_valkey.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"
@@ -41,7 +41,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"
 # shellcheck source=utils/lib_sxng_vite.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh"
 
-PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
+# add ./local dev tools from python (virtualenv), golang and nodejs
+PATH="${PY_ENV}/bin:${REPO_ROOT}/node_modules/.bin:${GOROOT}/bin:${GOPATH}/bin:${PATH}"
 
 # config
 
@@ -100,11 +101,27 @@ EOF
     static.help
     vite.help
     cat <<EOF
+dev.:
+  env: enter developer environment (or exec a command in)
 environment ...
   SEARXNG_VALKEY_URL : ${SEARXNG_VALKEY_URL}
 EOF
 }
 
+dev.env() {
+    go.env.dev
+    node.env.dev
+
+    export GOENV
+
+    if [ -z "$1" ]; then
+        export PS1="(dev.env)$ "
+        bash --norc --noprofile
+    else
+        "$@"
+    fi
+}
+
 
 if [ "$VERBOSE" = "1" ]; then
     SPHINX_VERBOSE="-v"

utils/lib_go.sh

@@ -1,211 +0,0 @@
-#!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
-# SPDX-License-Identifier: AGPL-3.0-or-later
-#
-# Tools to install and maintain golang [1] binaries & packages.
-#
-# [1] https://golang.org/doc/devel/release#policy
-#
-# A simple *helloworld* test with user 'my_user' :
-#
-#   sudo -H adduser my_user
-#   ./manage go.golang go1.17.3 my_user
-#   ./manage go.install github.com/go-training/helloworld@latest my_user
-#   ./manage go.bash my_user
-#   $ helloword
-#   Hello World!!
-#
-# Don't forget to remove 'my_user':  sudo -H deluser --remove-home my_user
-
-# shellcheck source=utils/lib.sh
-. /dev/null
-
-# configure golang environment
-# ----------------------------
-
-[[ -z "${GO_VERSION}" ]] && GO_VERSION="go1.17.3"
-
-GO_DL_URL="https://golang.org/dl"
-
-# implement go functions
-# -----------------------
-
-go.help(){
-    cat <<EOF
-go.:
-  ls        : list golang binary archives (stable)
-  golang    : (re-) install golang binary in user's \$HOME/local folder
-  install   : install go package in user's \$HOME/go-apps folder
-  bash      : start bash interpreter with golang environment sourced
-EOF
-}
-
-go.ls(){
-    python <<EOF
-import sys, json, requests
-resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
-for ver in json.loads(resp.text):
-    if not ver['stable']:
-        continue
-    for f in ver['files']:
-        if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:
-            continue
-        print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)
-EOF
-}
-
-go.ver_info(){
-
-    # print information about a golang distribution. To print filename
-    # sha256 and size of the archive that fits to your OS and host:
-    #
-    #   go.ver_info "${GO_VERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
-    #
-    # usage: go.ver_info <go-vers> <kind> <os> <arch> [filename|sha256|size]
-    #
-    # kind:  [archive|source|installer]
-    # os:    [darwin|freebsd|linux|windows]
-    # arch:  [amd64|arm64|386|armv6l|ppc64le|s390x]
-
-    python - "$@" <<EOF
-import sys, json, requests
-resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
-for ver in json.loads(resp.text):
-    if ver['version'] != sys.argv[1]:
-        continue
-    for f in ver['files']:
-        if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):
-            continue
-        for x in sys.argv[5:]:
-           print(f[x])
-        sys.exit(0)
-sys.exit(42)
-EOF
-}
-
-go.os() {
-  local OS
-  case "$(command uname -a)xx" in
-    Linux\ *) OS=linux ;;
-    Darwin\ *) OS=darwin ;;
-    FreeBSD\ *) OS=freebsd ;;
-    CYGWIN* | MSYS* | MINGW*) OS=windows ;;
-    *)  die 42 "OS is unknown: $(command uname -a)" ;;
-  esac
-  echo "${OS}"
-}
-
-go.arch() {
-    local ARCH
-    case "$(command uname -m)" in
-        "x86_64") ARCH=amd64 ;;
-        "aarch64") ARCH=arm64 ;;
-        "armv6" | "armv7l") ARCH=armv6l ;;
-        "armv8") ARCH=arm64 ;;
-        .*386.*) ARCH=386 ;;
-        ppc64*) ARCH=ppc64le ;;
-    *)  die 42 "ARCH is unknown: $(command uname -m)" ;;
-    esac
-    echo "${ARCH}"
-}
-
-go.golang() {
-
-    # install golang binary in user's $HOME/local folder:
-    #
-    #   go.golang ${GO_VERSION} ${SERVICE_USER}
-    #
-    # usage:  go.golang <go-vers> [<username>]
-
-    local version fname sha size user userpr
-    local buf=()
-
-    version="${1:-${GO_VERSION}}"
-    user="${2:-${USERNAME}}"
-    userpr="  ${_Yellow}|${user}|${_creset} "
-
-    rst_title "Install Go in ${user}'s HOME" section
-
-    mapfile -t buf < <(
-        go.ver_info "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size
-    )
-
-    if [ ${#buf[@]} -eq 0 ]; then
-        die 42 "can't find info of golang version: ${version}"
-    fi
-    fname="${buf[0]}"
-    sha="${buf[1]}"
-    size="$(numfmt --to=iec "${buf[2]}")"
-
-    info_msg "Download go binary ${fname} (${size}B)"
-    cache_download "${GO_DL_URL}/${fname}" "${fname}"
-
-    pushd "${CACHE}" &> /dev/null
-    echo "${sha}  ${fname}" > "${fname}.sha256"
-    if ! sha256sum -c "${fname}.sha256" >/dev/null; then
-        die 42 "downloaded file ${fname} checksum does not match"
-    else
-        info_msg "${fname} checksum OK"
-    fi
-    popd &> /dev/null
-
-    info_msg "install golang"
-    tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
-mkdir -p \$HOME/local
-rm -rf \$HOME/local/go
-tar -C \$HOME/local -xzf ${CACHE}/${fname}
-echo "export GOPATH=\$HOME/go-apps" > \$HOME/.go_env
-echo "export PATH=\$HOME/local/go/bin:\\\$GOPATH/bin:\\\$PATH" >> \$HOME/.go_env
-EOF
-    info_msg "test golang installation"
-    sudo -i -u "${user}" <<EOF
-source \$HOME/.go_env
-command -v go
-go version
-EOF
-}
-
-go.install() {
-
-    # install go package in user's $HOME/go-apps folder:
-    #
-    #   go.install github.com/go-training/helloworld@lates ${SERVICE_USER}
-    #
-    # usage:  go.install <package> [<username>]
-
-    local package user userpr
-
-    package="${1}"
-    user="${2:-${USERNAME}}"
-    userpr="  ${_Yellow}|${user}|${_creset} "
-
-    if [ -z "${package}" ]; then
-        die 42 "${FUNCNAME[0]}() - missing argument: <package>"
-    fi
-    tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
-source \$HOME/.go_env
-go install -v ${package}
-EOF
-}
-
-go.bash() {
-
-    # start bash interpreter with golang environment sourced
-    #
-    #   go.bash ${SERVICE_USER}
-    #
-    # usage:  go.bash [<username>]
-
-    local user
-    user="${1:-${USERNAME}}"
-    sudo -i -u "${user}" bash --init-file "~${user}/.go_env"
-}
-
-go.version(){
-    local user
-    user="${1:-${USERNAME}}"
-    sudo -i -u "${user}" <<EOF
-source \$HOME/.go_env
-go version | cut -d' ' -f 3
-EOF
-}

utils/lib_govm.sh

@@ -0,0 +1,226 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# Go versions manager to install and maintain golang [1] binaries & packages.
+#
+# [1] https://golang.org/doc/devel/release#policy
+
+# shellcheck source=utils/lib.sh
+. /dev/null
+
+# configure golang environment for go.vm
+# --------------------------------------
+
+_GO_DL_URL="https://go.dev/dl"
+
+GOVERSION="${GOVERSION:-go$(awk '/^go /{print $2}' "${REPO_ROOT}/go.mod")}"
+GOTOOLCHAIN=local
+
+GOROOT="${REPO_ROOT}/.govm/${GOVERSION}"
+GOENV="${GOROOT}/.config/go.env"
+GOVM_EXE="${GOROOT}/bin/go"
+
+GOPATH="${REPO_ROOT}/local/${GOVERSION}" # no support for multiple path names!!
+GOCACHE="${GOPATH}/.cache/go-build"
+GOMODCACHE="${GOPATH}/pkg/mod"
+
+# implement go functions
+# -----------------------
+
+go.help() {
+    cat <<EOF
+go:           GOROOT=${GOROOT}
+  install   : compiles and installs packages
+EOF
+}
+
+go.tool() {
+    # shortcut for "go tool .." in the Go environment
+    go.env.dev
+    "${GOVM_EXE}" tool "$@"
+}
+
+go.env.dev() {
+    if [ -z "$_GO_DEVTOOLS_INSTALLED" ]; then
+        build_msg INSTALL "[pkg.go.dev] ./go.mod: developer and CI tools"
+        go.tidy
+    else
+        go.vm.ensure
+        _GO_DEVTOOLS_INSTALLED=1
+    fi
+}
+
+go.tidy() {
+    go.vm.ensure
+    "${GOVM_EXE}" mod tidy
+    chmod -R u+w "${GOMODCACHE}"
+}
+
+go.clean() {
+    if ! go.vm.is_installed; then
+        build_msg CLEAN "[Go] not installed"
+        return 0
+    fi
+    build_msg CLEAN "[Go] drop folders ${GOROOT} and ${GOPATH}"
+    rm -rf "${GOROOT}" "${GOPATH}"
+}
+
+go.install() {
+    go.vm.ensure
+    GOENV="${GOENV}" "${GOVM_EXE}" install "$@"
+    # not sure why, but go installs some files without setting the write access
+    # for the file owner
+    chmod -R u+w "${GOMODCACHE}"
+}
+
+go.os() {
+    local OS
+    case "$(command uname -a)xx" in
+        Linux\ *) OS=linux ;;
+        Darwin\ *) OS=darwin ;;
+        FreeBSD\ *) OS=freebsd ;;
+        CYGWIN* | MSYS* | MINGW*) OS=windows ;;
+        *) die 42 "OS is unknown: $(command uname -a)" ;;
+    esac
+    echo "${OS}"
+}
+
+go.arch() {
+    local ARCH
+    case "$(command uname -m)" in
+        "x86_64") ARCH=amd64 ;;
+        "aarch64") ARCH=arm64 ;;
+        "armv6" | "armv7l") ARCH=armv6l ;;
+        "armv8") ARCH=arm64 ;;
+        .*386.*) ARCH=386 ;;
+        ppc64*) ARCH=ppc64le ;;
+        *) die 42 "ARCH is unknown: $(command uname -m)" ;;
+    esac
+    echo "${ARCH}"
+}
+
+# Go version management (go.vm)
+# -----------------------------
+
+go.vm.ensure() {
+    if ! go.vm.is_installed; then
+        # shellcheck disable=SC2119
+        go.vm.install
+    fi
+}
+
+go.vm.is_installed() {
+    # is true if "go" command is installed
+    [[ -f "${GOROOT}/bin/go" ]]
+}
+
+# shellcheck disable=SC2120
+go.vm.install() {
+
+    python -m pip install -U requests
+
+    # Go versions manager; to install Go at arbitrary place:
+    #
+    # usage:  go.vm.install <version> <dest>
+
+    local version dest fname sha size tmp
+    version="${1:-$GOVERSION}"
+    dest="${2:-$GOROOT}"
+
+    info_msg "Install Go in ${dest}"
+
+    # HINT: the python requirements needed by go.vm.version are taken from the
+    # developer environment. If it is not yet installed, install it now ..
+    pyenv.install
+
+    # fetch go version ..
+    local buf=()
+    mapfile -t buf < <(
+        go.vm.version "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size
+    )
+    if [ ${#buf[@]} -eq 0 ]; then
+        die 42 "can't find info of golang version: ${version}"
+    fi
+    fname="${buf[0]}"
+    sha="${buf[1]}"
+    size="$(numfmt --to=iec "${buf[2]}")"
+
+    info_msg "Download go binary ${fname} (${size}B)"
+    cache_download "${_GO_DL_URL}/${fname}" "${fname}"
+
+    pushd "${CACHE}" &>/dev/null
+    echo "${sha}  ${fname}" >"${fname}.sha256"
+    if ! sha256sum -c "${fname}.sha256" >/dev/null; then
+        die 42 "downloaded file ${fname} checksum does not match"
+    else
+        info_msg "${fname} checksum OK"
+    fi
+    popd &>/dev/null
+
+    info_msg "install golang"
+
+    tmp="$(mktemp -d)"
+    tar -C "${tmp}" -xzf "${CACHE}/${fname}"
+    rm -rf "${dest}"
+    mkdir -p "$(dirname "${dest}")"
+    mv "${tmp}/go" "${dest}"
+
+    mkdir -p "$(dirname "$GOENV")"
+    export GOENV
+
+    "${GOVM_EXE}" telemetry off
+    "${GOVM_EXE}" env -w \
+        GOBIN="$GOBIN" \
+        GOTOOLCHAIN="$GOTOOLCHAIN" \
+        GOCACHE="$GOCACHE" \
+        GOPATH="$GOPATH" \
+        GOMODCACHE="$GOMODCACHE"
+
+    mkdir -p "${GOMODCACHE}"
+}
+
+go.vm.list() {
+
+    # Go versions manager; list Go versions (stable)
+
+    python <<EOF
+import sys, json, requests
+resp = requests.get("${_GO_DL_URL}/?mode=json&include=all")
+for ver in json.loads(resp.text):
+    if not ver['stable']:
+        continue
+    for f in ver['files']:
+        if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:
+            continue
+        print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)
+EOF
+}
+
+go.vm.version() {
+
+    # Print information about a Go distribution. To print filename sha256 and
+    # size of the archive that fits to your OS and host:
+    #
+    #   go.ver_info "${GOVERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
+    #
+    # usage: go.vm.version <go-vers> <kind> <os> <arch> [filename|sha256|size]
+    #
+    # kind:  [archive|source|installer]
+    # os:    [darwin|freebsd|linux|windows]
+    # arch:  [amd64|arm64|386|armv6l|ppc64le|s390x]
+
+    python - "$@" <<EOF
+import sys, json, requests
+resp = requests.get("${_GO_DL_URL}/?mode=json&include=all")
+for ver in json.loads(resp.text):
+    if ver['version'] != sys.argv[1]:
+        continue
+    for f in ver['files']:
+        if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):
+            continue
+        for x in sys.argv[5:]:
+           print(f[x])
+        sys.exit(0)
+sys.exit(42)
+EOF
+}