diff --git a/.editorconfig b/.editorconfig index b466d6876df..fe29f21d746 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,6 +1,13 @@ -# top-most EditorConfig file root = true +[*.sh] +indent_style = space +indent_size = 4 +shell_variant = bash +binary_next_line = true +switch_case_indent = true +space_redirects = true + # Unix-style newlines with a newline ending every file [*] end_of_line = lf diff --git a/.github/workflows/ci-bash.yml b/.github/workflows/ci-bash.yml new file mode 100644 index 00000000000..5f4adf88cdb --- /dev/null +++ b/.github/workflows/ci-bash.yml @@ -0,0 +1,32 @@ +name: ci-bash + +on: + push: + tags: + - v* + branches: + - master + pull_request: + +permissions: + contents: read + +jobs: + bash-unit-tests: + name: bash-unit-tests + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@fe104658747b27e96e4f7e80cd0a94068e53901d # v2.16.1 + with: + egress-policy: audit + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Install shfmt + run: | + sudo apt-get update + sudo apt-get install -y shfmt + - name: Run bash validation + env: + BASH_TEST_JUNIT: "0" + run: make validate-bash diff --git a/.shellspec b/.shellspec new file mode 100644 index 00000000000..95eeea9746d --- /dev/null +++ b/.shellspec @@ -0,0 +1,4 @@ +--default-path test/bash/spec +--helperdir test/bash/spec +--shell bash +--execdir @project diff --git a/Makefile b/Makefile index b822a79cfca..44b13cc0919 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,12 @@ E2E_FLAGS ?= -test.v --ginkgo.vv --ginkgo.timeout 180m --ginkgo.flake-attempts=2 E2E_LABEL ?= !smoke&&!regressiontest GO_FLAGS ?= -tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper OC ?= oc +SHFMT ?= shfmt + +# Keep Go-templated dnsmasq hooks out of the raw shfmt list; the surrounding +# gotmpl wrappers are not valid shfmt input. +BASH_FMT_FILES := hack/unit-test-bash.sh $(wildcard test/bash/spec/*.sh test/bash/spec/support/*.sh) +BASH_FMT_FILES_ABS := $(abspath $(BASH_FMT_FILES)) # When Go is not installed on the host (e.g., CI Docker-only jobs), provide # safe fallback values. Without this guard, every $(shell go ...) call — @@ -386,6 +392,24 @@ unit-test-go: $(GOTESTSUM) unit-test-go-coverpkg: $(GOTESTSUM) $(GOTESTSUM) --format pkgname --junitfile report.xml -- -coverpkg=./... -coverprofile=cover_coverpkg.out ./... +.PHONY: unit-test-bash +unit-test-bash: + bash ./hack/unit-test-bash.sh + +.PHONY: fmt-bash +fmt-bash: ## Format bash framework files using shfmt and .editorconfig + $(SHFMT) -w $(BASH_FMT_FILES_ABS) + +.PHONY: validate-fmt-bash +validate-fmt-bash: ## Verify bash framework files are formatted + if ! $(SHFMT) -d $(BASH_FMT_FILES_ABS); then \ + echo "You need to run 'make fmt-bash' to update bash formatting and commit the changes"; \ + exit 1; \ + fi + +.PHONY: validate-bash +validate-bash: validate-fmt-bash unit-test-bash ## Validate bash formatting and unit tests + .PHONY: fmt fmt: $(GOLANGCI_LINT) ## Format Go source files using golangci-lint formatters (gci, gofumpt) $(GOLANGCI_LINT) fmt diff --git a/docs/testing.md b/docs/testing.md index 1ce19736600..bf493fb928d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -24,6 +24,14 @@ To run python client and `az aro` CLI tests: make test-python ``` +To run bash unit tests for the critical VMSS bootstrap, recovery, and dnsmasq shell assets: + +```bash +make unit-test-bash +``` + +The bash suite uses a pinned `ShellSpec` container image via `hack/unit-test-bash.sh`, so the same command works on Linux and macOS as long as Docker or Podman is available. On macOS, the runner defaults to `linux/amd64` for compatibility with the upstream ShellSpec image. + To run Go linting tasks (requires [golanglint-ci](https://golangci-lint.run/usage/install/) to be installed): ```bash diff --git a/hack/unit-test-bash.sh b/hack/unit-test-bash.sh new file mode 100644 index 00000000000..d2c53f01870 --- /dev/null +++ b/hack/unit-test-bash.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly repo_root +readonly shellspec_image="${BASH_TEST_IMAGE:-docker.io/shellspec/shellspec-debian:0.28.1}" +readonly default_report_base="${TMPDIR:-/tmp}/aro-bash-test-report" + +# Reject report paths that could make host-side cleanup delete the wrong tree. +validate_report_dir() { + local report_dir="$1" + local tmpdir="${TMPDIR:-/tmp}" + local path_segment + + tmpdir="${tmpdir%/}" + + if [[ -z "${report_dir}" || "${report_dir}" != /* || -z "${report_dir//\//}" ]]; then + echo "error: refusing unsafe report directory '${report_dir}'" >&2 + return 1 + fi + + # Only allow report directories under the temp dir with the expected prefix. + case "${report_dir}" in + "${tmpdir}"/aro-bash-test-report*) + ;; + *) + echo "error: refusing unsafe report directory '${report_dir}'" >&2 + return 1 + ;; + esac + + IFS='/' read -r -a path_segments <<< "${report_dir#/}" + for path_segment in "${path_segments[@]}"; do + if [[ "${path_segment}" == "." || "${path_segment}" == ".." ]]; then + echo "error: refusing unsafe report directory '${report_dir}'" >&2 + return 1 + fi + done +} + +# Recreate the host report directory that ShellSpec writes into from the container. +prepare_report_dir() { + local report_dir + report_dir="${BASH_TEST_REPORT_DIR:-${default_report_base}}" + validate_report_dir "${report_dir}" || return 1 + rm -rf -- "${report_dir}" + mkdir -p -- "${report_dir}" + printf '%s\n' "${report_dir}" +} + +# Treat an engine as usable only when its info command succeeds. +container_engine_usable() { + "$1" info > /dev/null 2>&1 +} + +# Detect podman-docker wrappers so we can apply Podman-specific run flags. +docker_is_podman() { + if ! command -v docker > /dev/null 2>&1; then + return 1 + fi + + local docker_path podman_path docker_version docker_info + + # Same inode: `docker` is a symlink/hardlink to the podman binary. + docker_path="$(type -P docker || true)" + podman_path="$(type -P podman || true)" + if [[ -n "${docker_path}" && -n "${podman_path}" && "${docker_path}" -ef "${podman_path}" ]]; then + return 0 + fi + + # Authoritative signals that do not depend on the podman-docker wrapper's + # "Emulate Docker CLI using podman" stderr message, which is suppressed when + # /etc/containers/nodocker exists. + docker_version="$(docker version --format '{{.Client.Name}}' 2> /dev/null || true)" + if [[ "${docker_version}" == *[Pp]odman* ]]; then + return 0 + fi + + docker_version="$(docker --version 2> /dev/null || true)" + if [[ "${docker_version}" == *[Pp]odman* ]]; then + return 0 + fi + + # Fallback for older podman-docker wrappers without /etc/containers/nodocker. + docker_info="$(docker info 2>&1 || true)" + [[ "${docker_info}" == *"Emulate Docker CLI using podman"* ]] +} + +# Prefer docker when available, unless it is really Podman underneath. +detect_container_engine() { + if command -v docker > /dev/null 2>&1 && container_engine_usable docker; then + if docker_is_podman; then + echo podman + else + echo docker + fi + return 0 + fi + + if command -v podman > /dev/null 2>&1 && container_engine_usable podman; then + echo podman + return 0 + fi + + echo "error: a working docker or podman daemon is required to run bash tests" >&2 + return 1 +} + +# Keep this destructive suite serial unless the caller explicitly opts in. +detect_jobs() { + if [[ -n "${BASH_TEST_JOBS:-}" ]]; then + echo "${BASH_TEST_JOBS}" + return 0 + fi + echo 1 +} + +# Force linux/amd64 only on macOS hosts that default to Apple Silicon containers. +detect_platform() { + if [[ -n "${BASH_TEST_PLATFORM:-}" ]]; then + echo "${BASH_TEST_PLATFORM}" + return 0 + fi + + if [[ "$(uname -s)" == "Darwin" ]]; then + echo "linux/amd64" + fi +} + +# Assemble engine-specific mounts and run ShellSpec inside the container. +main() { + local engine jobs mount_arg report_dir report_mount + engine="$(detect_container_engine)" + jobs="$(detect_jobs)" + report_dir="$(prepare_report_dir)" || return 1 + + local -a engine_args=() + if [[ "${engine}" == "podman" ]]; then + mount_arg="${repo_root}:/work:Z" + report_mount="${report_dir}:/work/.bash-test-report:Z" + engine_args+=("--userns=keep-id:uid=0,gid=0") + else + mount_arg="${repo_root}:/work" + report_mount="${report_dir}:/work/.bash-test-report" + fi + + local -a platform_args=() + local platform + platform="$(detect_platform)" + if [[ -n "${platform}" ]]; then + platform_args=(--platform "${platform}") + fi + + local junit_enabled="${BASH_TEST_JUNIT:-${CI:-0}}" + local -a report_args=() + if [[ "${junit_enabled}" == "true" || "${junit_enabled}" == "1" ]]; then + report_args=(--output junit --reportdir /work/.bash-test-report) + echo "ShellSpec JUnit report directory: ${report_dir}" + fi + + echo "Running ShellSpec with ${engine} using image ${shellspec_image}" + + "${engine}" run --rm \ + "${platform_args[@]}" \ + "${engine_args[@]}" \ + -v "${mount_arg}" \ + -v "${report_mount}" \ + -w /work \ + "${shellspec_image}" \ + --jobs "${jobs}" \ + --format documentation \ + "${report_args[@]}" \ + "$@" +} + +main "$@" diff --git a/pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl b/pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl index 0d98a139d0d..540608d9db1 100644 --- a/pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl +++ b/pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl @@ -1,5 +1,5 @@ {{ define "99-dnsmasq-restart" }} -#!/bin/sh +#!/bin/bash # This is a NetworkManager dispatcher script to restart dnsmasq # in the event of a network interface change (e. g. host servicing event https://learn.microsoft.com/en-us/azure/developer/intro/hosting-apps-on-azure) # this will restart dnsmasq, reapplying our /etc/resolv.conf file and overwriting any modifications made by NetworkManager diff --git a/test/bash/spec/backupandfixetcd_spec.sh b/test/bash/spec/backupandfixetcd_spec.sh new file mode 100644 index 00000000000..711bdb534ec --- /dev/null +++ b/test/bash/spec/backupandfixetcd_spec.sh @@ -0,0 +1,84 @@ +Describe 'backupandfixetcd.sh' +Include ./test/bash/spec/support/helpers.sh +script_path="${REPO_ROOT}/pkg/frontend/scripts/backupandfixetcd.sh" + +run_backup_mode() { + ensure_test_workspace + mkdir -p /etc/kubernetes/manifests /var/lib/etcd + printf 'manifest-data' > /etc/kubernetes/manifests/etcd-pod.yaml + printf 'etcd-data' > /var/lib/etcd/member + + BACKUP=1 FIX_PEERS='' bash "${script_path}" + + backup_dir="$(echo /var/lib/etcd-backup-*)" + printf '%s\n' "${backup_dir}" + printf '%s\n' "$(cat "${backup_dir}/etcd-pod.yaml")" + printf '%s\n' "$(cat "${backup_dir}/etcd/member")" + if backup_paths_moved; then + printf '%s\n' 'moved=yes' + else + printf '%s\n' 'moved=no' + fi +} + +backup_paths_moved() { + [ ! -e /var/lib/etcd ] && [ ! -e /etc/kubernetes/manifests/etcd-pod.yaml ] +} + +run_fix_peers_mode() { + ensure_test_workspace + + append_mock_command oc ' + printf "oc %s\n" "$*" >> "${CALL_LOG}" + if [[ "$*" == *"member list"* ]]; then + printf "%s\n" "{\"members\":[{\"name\":\"master-0\",\"ID\":\"member-123\"}]}" + fi + ' + append_mock_command jq 'printf "%s\n" "member-123"' + cp "${MOCK_BIN}/oc" /host/usr/bin/oc + cp "${MOCK_BIN}/jq" /host/usr/bin/jq + + FIX_PEERS=1 \ + BACKUP='' \ + PEER_PODS="etcd-0 etcd-1" \ + DEGRADED_NODE="master-0" \ + bash "${script_path}" + + cat "${CALL_LOG}" +} + +verify_fix_peers_host_mocks_are_cleaned() { + ensure_test_workspace + + append_mock_command oc ':' + append_mock_command jq ':' + cp "${MOCK_BIN}/oc" /host/usr/bin/oc + cp "${MOCK_BIN}/jq" /host/usr/bin/jq + + reset_absolute_test_state + + [ ! -e /host/usr/bin/oc ] && [ ! -e /host/usr/bin/jq ] +} + +It 'backs up etcd manifests and data into a timestamped directory' +When call run_backup_mode +The status should be success +The output should include '/var/lib/etcd-backup-' +The output should include 'manifest-data' +The output should include 'etcd-data' +The output should include 'moved=yes' +End + +It 'removes degraded peer members when FIX_PEERS is set' +When call run_fix_peers_mode +The status should be success +The output should include 'Starting peer etcd member removal' +The output should include 'oc rsh -n openshift-etcd -c etcdctl pod/etcd-0 etcdctl member remove member-123' +The output should include 'oc rsh -n openshift-etcd -c etcdctl pod/etcd-1 etcdctl member remove member-123' +End + +It 'cleans fix-peers host mocks during helper reset' +When run verify_fix_peers_host_mocks_are_cleaned +The status should be success +End +End diff --git a/test/bash/spec/dnsmasq_templates_spec.sh b/test/bash/spec/dnsmasq_templates_spec.sh new file mode 100644 index 00000000000..670241adfd7 --- /dev/null +++ b/test/bash/spec/dnsmasq_templates_spec.sh @@ -0,0 +1,104 @@ +Describe 'dnsmasq shell templates' +Include ./test/bash/spec/support/helpers.sh +pre_template="${REPO_ROOT}/pkg/operator/controllers/dnsmasq/scripts/aro-dnsmasq-pre.sh.gotmpl" +restart_template="${REPO_ROOT}/pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl" + +render_pre_template() { + extract_template_body "${pre_template}" +} + +render_restart_template() { + extract_template_body "${restart_template}" +} + +run_restart_template_for_event() { + ensure_test_workspace + local interface="$1" + local action="$2" + local rendered_script="${TEST_ROOT}/99-dnsmasq-restart.sh" + + extract_template_body "${restart_template}" > "${rendered_script}" + chmod +x "${rendered_script}" + + write_file_contents /etc/resolv.conf.dnsmasq 'nameserver 10.0.0.2' + append_mock_command logger 'printf "logger %s\n" "$*" >> "${CALL_LOG}"' + append_mock_command systemctl 'printf "systemctl %s\n" "$*" >> "${CALL_LOG}"' + + CONNECTIVITY_STATE="full" bash "${rendered_script}" "${interface}" "${action}" > /dev/null + + cat "${CALL_LOG}" +} + +run_restart_template_for_irrelevant_event() { + run_restart_template_for_event "lo" "up" +} + +render_template_with_internal_blank_lines() { + ensure_test_workspace + + local template="${TEST_ROOT}/blank-lines.gotmpl" + cat > "${template}" << 'EOF' +{{ define "blank-lines" }} + +#!/bin/bash + +echo first + +echo second + +{{ end }} + +EOF + + extract_template_body "${template}" +} + +It 'renders the pre hook with dnsmasq and NetworkManager updates' +When call render_pre_template +The status should be success +The output should start with '#!/bin/bash' +The output should include '/etc/resolv.conf.dnsmasq' +The output should include '/etc/NetworkManager/conf.d/aro-dns.conf' +The output should include '/usr/bin/nmcli general reload conf' +The output should include '/usr/bin/nmcli general reload dns-rc' +End + +It 'renders the restart hook with dnsmasq restart logic for interface changes' +When call render_restart_template +The status should be success +The output should start with '#!/bin/bash' +The output should include 'logger -i "$0" -t '"'"'99-DNSMASQ-RESTART SCRIPT'"'"'' +The output should include 'systemctl try-restart dnsmasq --wait' +The output should include '[[ $interface == eth* && $action == "up" ]]' +The output should include '[[ $interface == enP* && $action == "down" ]]' +End + +It 'restarts dnsmasq for matching interface transitions' +When call run_restart_template_for_event "eth0" "up" +The status should be success +The output should include 'systemctl try-restart dnsmasq --wait' +The output should include 'dnsmasq successfully restarted' +End + +It 'skips dnsmasq restart for unrelated interface transitions' +When call run_restart_template_for_irrelevant_event +The status should be success +The output should eq '' +End + +It 'preserves internal blank lines while stripping template delimiters' +expected_blank_lines="$( + cat << 'EOF' +echo first + +echo second +EOF +)" + +When call render_template_with_internal_blank_lines +The status should be success +The output should start with '#!/bin/bash' +The output should include "${expected_blank_lines}" +The output should end with 'echo second' +End +End diff --git a/test/bash/spec/spec_helper.sh b/test/bash/spec/spec_helper.sh new file mode 100644 index 00000000000..6f2af181d77 --- /dev/null +++ b/test/bash/spec/spec_helper.sh @@ -0,0 +1,45 @@ +set -eu + +is_container_test_environment() { + [ "${container:-}" = "docker" ] \ + || [ "${container:-}" = "podman" ] \ + || [ -f "/.dockerenv" ] \ + || [ -f "/run/.containerenv" ] +} + +spec_helper_precheck() { + minimum_version "0.28.0" + if [ "$SHELL_TYPE" != "bash" ]; then + abort "Only bash is supported." + fi + + if [ "${BASH_TEST_ALLOW_HOST:-0}" != "1" ] && ! is_container_test_environment; then + abort "Refusing to run destructive bash tests on the host. Run inside a container or set BASH_TEST_ALLOW_HOST=1 to opt in." + fi +} + +spec_helper_loaded() { + # shellcheck source=test/bash/spec/support/helpers.sh + . "${SHELLSPEC_HELPERDIR}/support/helpers.sh" +} + +spec_helper_configure() { + before_each 'setup_test_workspace' + after_each 'cleanup_test_workspace' +} + +setup_test_workspace() { + : +} + +cleanup_test_workspace() { + reset_absolute_test_state + rm -f /etc/ssh/sshd_config + if [ "${TEST_ROOT:-}" ]; then + case "$TEST_ROOT" in + "${TMPDIR:-/tmp}"/aro-bash-tests.*) + rm -rf -- "$TEST_ROOT" + ;; + esac + fi +} diff --git a/test/bash/spec/support/helpers.sh b/test/bash/spec/support/helpers.sh new file mode 100644 index 00000000000..1c85f69f253 --- /dev/null +++ b/test/bash/spec/support/helpers.sh @@ -0,0 +1,266 @@ +REPO_ROOT="${REPO_ROOT:-$(cd "${SHELLSPEC_HELPERDIR}/../../.." && pwd)}" +export REPO_ROOT + +ensure_test_workspace() { + if [ -n "${TEST_ROOT:-}" ] && [ -d "${TEST_ROOT}" ]; then + return 0 + fi + + TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/aro-bash-tests.XXXXXX")" + SCRIPT_FIXTURE_DIR="${TEST_ROOT}/scripts" + MOCK_BIN="${TEST_ROOT}/bin" + CALL_LOG="${TEST_ROOT}/calls.log" + ORIGINAL_PATH="${ORIGINAL_PATH:-$PATH}" + + mkdir -p "${SCRIPT_FIXTURE_DIR}" "${MOCK_BIN}" + mkdir -p /etc/ssh /etc/sysconfig /etc/systemd/system /etc/cron.weekly /etc/cron.daily + mkdir -p /etc/NetworkManager/conf.d /etc/kubernetes/manifests /var/lib /host/usr/bin /host/usr/lib64 /usr/lib64 + : > "${CALL_LOG}" + + PATH="${MOCK_BIN}:${ORIGINAL_PATH}" + export TEST_ROOT SCRIPT_FIXTURE_DIR MOCK_BIN CALL_LOG ORIGINAL_PATH PATH +} + +reset_absolute_test_state() { + rm -rf /etc/proxy + rm -rf /root/.docker + rm -rf /var/lib/etcd + rm -rf /var/lib/etcd-backup-* + rm -f /etc/sysconfig/aro-gateway + rm -f /etc/sysconfig/aro-rp + rm -f /etc/sysconfig/proxy + rm -f /etc/systemd/system/aro-gateway.service + rm -f /etc/systemd/system/aro-rp.service + rm -f /etc/systemd/system/proxy.service + rm -f /etc/cron.weekly/pull-image + rm -f /etc/cron.weekly/yumupdate + rm -f /etc/cron.daily/restart-proxy + rm -f /etc/kubernetes/manifests/etcd-pod.yaml + rm -f /etc/resolv.conf.dnsmasq + rm -f /etc/NetworkManager/conf.d/aro-dns.conf + rm -f /host/usr/bin/jq + rm -f /host/usr/bin/oc + + local lib + for lib in /usr/lib64/libjq.so.1 /usr/lib64/libonig.so.5; do + if [ -L "${lib}" ] && [ "$(readlink "${lib}")" = "/host${lib}" ]; then + rm -f -- "${lib}" + fi + done + +copy_fixture() { + local source_path="$1" + local destination_name="$2" + + ensure_test_workspace + cp "${source_path}" "${SCRIPT_FIXTURE_DIR}/${destination_name}" + chmod +x "${SCRIPT_FIXTURE_DIR}/${destination_name}" + echo "${SCRIPT_FIXTURE_DIR}/${destination_name}" +} + +write_file_contents() { + local target_path="$1" + shift + + ensure_test_workspace + mkdir -p "$(dirname "${target_path}")" + cat > "${target_path}" << EOF +$* +EOF +} + +append_mock_command() { + local name="$1" + shift + + ensure_test_workspace + cat > "${MOCK_BIN}/${name}" << EOF +#!/bin/bash +set -eu +$* +EOF + chmod +x "${MOCK_BIN}/${name}" +} + +write_vmss_util_stub() { + ensure_test_workspace + cat > "${SCRIPT_FIXTURE_DIR}/util.sh" << 'EOF' +#!/bin/bash +set -eu + +declare -r role_gateway="gateway" +declare -r role_rp="rp" +declare -r role_devproxy="devproxy" + +_log() { + printf '%s\n' "$*" >> "${CALL_LOG}" +} + +_log_block() { + local label="$1" + local contents="$2" + { + printf '%s<> "${CALL_LOG}" +} + +create_required_dirs() { + _log "create_required_dirs" +} + +configure_sshd() { + _log "configure_sshd" +} + +configure_rpm_repos() { + local -n wait_ref="$1" + _log "configure_rpm_repos wait=${wait_ref} retries=${2:-}" +} + +dnf_update_pkgs() { + local -n excludes_ref="$1" + local -n wait_ref="$2" + _log "dnf_update_pkgs excludes=${excludes_ref[*]} wait=${wait_ref} retries=${3:-}" +} + +dnf_install_pkgs() { + local -n packages_ref="$1" + local -n wait_ref="$2" + _log "dnf_install_pkgs packages=${packages_ref[*]} wait=${wait_ref} retries=${3:-}" +} + +fips_configure() { + _log "fips_configure" +} + +host_mem_mib() { + printf '%s\n' "16384" +} + +configure_logrotate() { + _log "configure_logrotate" +} + +pull_container_images() { + local -n images_ref="$1" + _log "pull_container_images keys=${!images_ref[*]}" +} + +firewalld_configure() { + local -n ports_ref="$1" + _log "firewalld_configure ports=${ports_ref[*]}" +} + +configure_vmss_aro_services() { + local -n role_ref="$1" + local -n images_ref="$2" + local -n configs_ref="$3" + _log "configure_vmss_aro_services role=${role_ref} images=${!images_ref[*]} configs=${!configs_ref[*]}" + + local config_name + for config_name in "${!configs_ref[@]}"; do + case "${config_name}" in + rp_config|gateway_config|gateway_otel_collector|azuremonitor_tenant) + local -n config_ref="${configs_ref[$config_name]}" + _log_block "${config_name}" "${config_ref}" + ;; + esac + done +} + +enable_services() { + local -n services_ref="$1" + _log "enable_services services=${services_ref[*]}" +} + +reboot_vm() { + _log "reboot_vm" +} +EOF + chmod +x "${SCRIPT_FIXTURE_DIR}/util.sh" +} + +set_vmss_environment() { + export AZURECLOUDNAME="AzurePublicCloud" + export RPIMAGE="example.azurecr.io/aro:latest" + export MDMIMAGE="example.azurecr.io/mdm:latest" + export MISEIMAGE="example.azurecr.io/mise:latest" + export OTELIMAGE="example.azurecr.io/otel:latest" + export FLUENTBITIMAGE="example.azurecr.io/fluentbit:latest" + export GATEWAYOTELCOLLECTORIMAGE="example.azurecr.io/gateway-otel-collector:latest" + export ACRRESOURCEID="/subscriptions/test/resourceGroups/example/providers/Microsoft.ContainerRegistry/registries/example" + export DATABASEACCOUNTNAME="db-account" + export RPMDMACCOUNT="rp-mdm-account" + export GATEWAYDOMAINS="gateway.example.com" + export GATEWAYFEATURES="alpha,beta" + export GATEWAYMDSDCONFIGVERSION="1" + export GATEWAYOTELKUSTOINGESTIONENDPOINT="https://example.kusto.windows.net" + export RPLOGLEVEL="debug" + export GATEWAYLOGLEVEL="info" + export RPMDSDCONFIGVERSION="2" + export ADMINAPICLIENTCERTCOMMONNAME="admin" + export ARMAPICLIENTCERTCOMMONNAME="arm" + export ARMCLIENTID="arm-client" + export FPCLIENTID="fp-client" + export FPSERVICEPRINCIPALID="fp-sp" + export GATEWAYCLIENTID="gateway-client" + export CLUSTERMDMACCOUNT="cluster-mdm" + export CLUSTERMDSDACCOUNT="cluster-mdsd" + export CLUSTERMDSDCONFIGVERSION="3" + export OTELCLUSTERMDSDCONFIGVERSION="4" + export CLUSTERMDSDNAMESPACE="cluster-ns" + export LOCATION="westeurope" + export CLUSTERPARENTDOMAINNAME="aroapp.io" + export GATEWAYRESOURCEGROUPNAME="gateway-rg" + export GATEWAYUSERASSIGNEDIDENTITYRESOURCEID="/subscriptions/test/resourceGroups/example/providers/Microsoft.ManagedIdentity/userAssignedIdentities/gateway" + export KEYVAULTPREFIX="kv-prefix" + export MDSDENVIRONMENT="prod" + export RPFEATURES="feature-a" + export CLUSTERSINSTALLVIAHIVE="false" + export CLUSTERDEFAULTINSTALLERPULLSPEC="installer:latest" + export CLUSTERSADOPTBYHIVE="false" + export RPPARENTDOMAINNAME="rp-parent.example.com" + export OIDCSTORAGEACCOUNTNAME="oidcstorage" + export OTELAUDITQUEUESIZE="10" + export MSIRPENDPOINT="https://msi.example.com" + export ENVIRONMENT="test" + export PROXYIMAGE="example.azurecr.io/proxy:latest" + export PROXYIMAGEAUTH="cHJveHk6YXV0aA==" + export PROXYCERT="Y2VydA==" + export PROXYKEY="a2V5" + export PROXYCLIENTCERT="Y2xpZW50" +} + +extract_template_body() { + local line + local start end i + local -a lines=() + + while IFS= read -r line || [ -n "${line}" ]; do + if [[ "${line}" =~ ^[[:space:]]*\{\{[[:space:]]*define[[:space:]].*\}\}[[:space:]]*$ ]]; then + continue + fi + + if [[ "${line}" =~ ^[[:space:]]*\{\{[[:space:]]*end[[:space:]]*\}\}[[:space:]]*$ ]]; then + continue + fi + + lines+=("${line}") + done < "$1" + + start=0 + while [ "${start}" -lt "${#lines[@]}" ] && [[ "${lines[$start]}" =~ ^[[:space:]]*$ ]]; do + start=$((start + 1)) + done + + end=$((${#lines[@]} - 1)) + while [ "${end}" -ge "${start}" ] && [[ "${lines[$end]}" =~ ^[[:space:]]*$ ]]; do + end=$((end - 1)) + done + + for ((i = start; i <= end; i++)); do + printf '%s\n' "${lines[$i]}" + done +} diff --git a/test/bash/spec/unit_test_bash_runner_spec.sh b/test/bash/spec/unit_test_bash_runner_spec.sh new file mode 100644 index 00000000000..078351126c2 --- /dev/null +++ b/test/bash/spec/unit_test_bash_runner_spec.sh @@ -0,0 +1,130 @@ +Describe 'bash unit test runner' +Include ./test/bash/spec/support/helpers.sh + +run_runner_with_podman_docker() { + ensure_test_workspace + + append_mock_command podman ' +case "${1:-}" in + info) + if [ "${0##*/}" = "docker" ]; then + printf "%s\n" "Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg." >&2 + fi + exit 0 + ;; + run) + printf "%s %s\n" "${0##*/}" "$*" + exit 0 + ;; +esac +' + + ln -s "${MOCK_BIN}/podman" "${MOCK_BIN}/docker" + + ( + PATH="${MOCK_BIN}:${ORIGINAL_PATH}" \ + BASH_TEST_REPORT_DIR="${TEST_ROOT}/report" \ + bash "${REPO_ROOT}/hack/unit-test-bash.sh" + ) 2>&1 +} + +run_runner_with_silenced_podman_docker() { + ensure_test_workspace + + # Mimic a podman-docker wrapper that is a *separate* script (not a symlink + # to podman) and whose "Emulate Docker CLI using podman" info message is + # silenced by /etc/containers/nodocker. Identity is only discoverable via + # `docker version`/`docker --version`. + append_mock_command podman ' +case "${1:-}" in + run) + printf "%s %s\n" "${0##*/}" "$*" + exit 0 + ;; +esac +' + + append_mock_command docker ' +case "${1:-}" in + info) + exit 0 + ;; + version) + printf "%s\n" "Podman Engine" + exit 0 + ;; + --version) + printf "%s\n" "podman version 5.8.3" + exit 0 + ;; +esac +' + + ( + PATH="${MOCK_BIN}:${ORIGINAL_PATH}" \ + BASH_TEST_REPORT_DIR="${TEST_ROOT}/report" \ + bash "${REPO_ROOT}/hack/unit-test-bash.sh" + ) 2>&1 +} + +run_runner_with_report_dir() { + ensure_test_workspace + + append_mock_command docker ' +case "${1:-}" in + info) + exit 0 + ;; + run) + printf "%s %s\n" "${0##*/}" "$*" + exit 0 + ;; +esac +' + + ( + PATH="${MOCK_BIN}:${ORIGINAL_PATH}" \ + BASH_TEST_REPORT_DIR="$1" \ + bash "${REPO_ROOT}/hack/unit-test-bash.sh" + ) 2>&1 +} + +run_runner_with_root_report_dir() { + run_runner_with_report_dir "/" +} + +run_runner_with_relative_report_dir() { + run_runner_with_report_dir "." +} + +It 'treats podman-docker as podman' +When call run_runner_with_podman_docker +The status should be success +The output should include 'Running ShellSpec with podman' +The output should include 'docker.io/shellspec/shellspec-debian:0.28.1' +The output should include 'podman run --rm' +The output should include '--userns=keep-id:uid=0,gid=0' +The output should include ':/work:Z' +End + +It 'detects podman-docker when the emulate message is silenced' +When call run_runner_with_silenced_podman_docker +The status should be success +The output should include 'Running ShellSpec with podman' +The output should include 'podman run --rm' +The output should include '--userns=keep-id:uid=0,gid=0' +The output should include ':/work:Z' +End + +It 'rejects root report directories' +When run run_runner_with_root_report_dir +The status should be failure +The output should include 'refusing unsafe report directory' +End + +It 'rejects relative report directories' +When run run_runner_with_relative_report_dir +The status should be failure +The output should include 'refusing unsafe report directory' +End +End diff --git a/test/bash/spec/vmss_entrypoints_spec.sh b/test/bash/spec/vmss_entrypoints_spec.sh new file mode 100644 index 00000000000..6f5ac6c4576 --- /dev/null +++ b/test/bash/spec/vmss_entrypoints_spec.sh @@ -0,0 +1,88 @@ +Describe 'VMSS entrypoint scripts' +Include ./test/bash/spec/support/helpers.sh +BeforeEach 'set_vmss_environment' + +run_vmss_script() { + ensure_test_workspace + local source_path="$1" + local destination_name="$2" + + copy_fixture "${source_path}" "${destination_name}" > /dev/null + write_vmss_util_stub + + ( + cd "${SCRIPT_FIXTURE_DIR}" + bash "./${destination_name}" > /dev/null + ) + + cat "${CALL_LOG}" +} + +run_devproxy_script() { + ensure_test_workspace + copy_fixture "${REPO_ROOT}/pkg/deploy/generator/scripts/devProxyVMSS.sh" "devProxyVMSS.sh" > /dev/null + + append_mock_command tdnf 'printf "tdnf %s\n" "$*" >> "${CALL_LOG}"' + append_mock_command systemctl 'printf "systemctl %s\n" "$*" >> "${CALL_LOG}"' + append_mock_command docker 'printf "docker %s\n" "$*" >> "${CALL_LOG}"' + append_mock_command sleep 'printf "sleep %s\n" "$*" >> "${CALL_LOG}"' + append_mock_command reboot 'printf "reboot\n" >> "${CALL_LOG}"' + + ( + cd "${SCRIPT_FIXTURE_DIR}" + bash ./devProxyVMSS.sh > /dev/null + ) + + cat "${CALL_LOG}" + printf '\n--proxy-env--\n' + cat /etc/sysconfig/proxy + printf '\n--proxy-service--\n' + cat /etc/systemd/system/proxy.service + printf '\n--pull-image--\n' + cat /etc/cron.weekly/pull-image + printf '\n--restart-proxy--\n' + cat /etc/cron.daily/restart-proxy +} + +It 'runs rpVMSS through the shared utility layer' +When call run_vmss_script "${REPO_ROOT}/pkg/deploy/generator/scripts/rpVMSS.sh" "rpVMSS.sh" +The status should be success +The output should include 'create_required_dirs' +The output should include 'configure_sshd' +The output should include 'dnf_install_pkgs packages=azure-cli azure-mdsd podman podman-docker openssl-perl python3 firewalld grubby dracut-fips' +The output should include 'configure_vmss_aro_services role=rp' +The output should include "MISE_ADDRESS='http://10.88.0.5:5000'" +The output should include "OIDC_AFD_ENDPOINT='westeurope.oic.rp-parent.example.com'" +The output should include "OTEL_AUDIT_QUEUE_SIZE='10'" +The output should include 'enable_services services=aro-mise aro-monitor aro-otel-collector aro-portal aro-mimo-actuator aro-mimo-scheduler aro-rp' +The output should include 'reboot_vm' +End + +It 'runs gatewayVMSS through the shared utility layer' +When call run_vmss_script "${REPO_ROOT}/pkg/deploy/generator/scripts/gatewayVMSS.sh" "gatewayVMSS.sh" +The status should be success +The output should include 'create_required_dirs' +The output should include 'configure_sshd' +The output should include 'dnf_install_pkgs packages=azure-cli azure-mdsd podman podman-docker openssl-perl python3 firewalld grubby dracut-fips' +The output should include 'configure_vmss_aro_services role=gateway' +The output should include "GATEWAY_FEATURES='alpha,beta'" +The output should include 'cluster_uri: "https://example.kusto.windows.net"' +The output should include 'limit_mib: 916' +The output should include 'spike_limit_mib: 109' +The output should include 'download-gateway-otel-credentials.timer' +The output should include 'enable_services services=aro-gateway azsecd mdsd mdm chronyd fluentbit' +The output should include 'reboot_vm' +End + +It 'writes the devproxy service and maintenance scripts' +When call run_devproxy_script +The status should be success +The output should include 'tdnf install -y moby-engine moby-cli' +The output should include 'systemctl enable docker' +The output should include 'docker pull example.azurecr.io/proxy:latest' +The output should include "PROXY_IMAGE='example.azurecr.io/proxy:latest'" +The output should include 'ExecStart=/usr/bin/docker run --rm --name %n -p 443:8443 -v /etc/proxy:/secrets $PROXY_IMAGE' +The output should include 'docker pull $PROXYIMAGE' +The output should include 'systemctl restart proxy.service' +End +End diff --git a/test/bash/spec/vmss_helpers_spec.sh b/test/bash/spec/vmss_helpers_spec.sh new file mode 100644 index 00000000000..0880a4115c9 --- /dev/null +++ b/test/bash/spec/vmss_helpers_spec.sh @@ -0,0 +1,215 @@ +Describe 'VMSS helper scripts' +Include ./test/bash/spec/support/helpers.sh +Include ./pkg/deploy/generator/scripts/util-common.sh +Include ./pkg/deploy/generator/scripts/util-packages.sh +Include ./pkg/deploy/generator/scripts/util-system.sh +Include ./pkg/deploy/generator/scripts/util-services.sh + +BeforeEach 'set_vmss_environment' + +Describe 'util-common.sh' +write_file_example() { + ensure_test_workspace + local target_file="${TEST_ROOT}/write-file.txt" + local initial_text="first" + local appended_text="second" + + write_file target_file initial_text true + write_file target_file appended_text false + + cat "${target_file}" +} + +retry_until_success_example() { + ensure_test_workspace + local counter_file="${TEST_ROOT}/retry-count" + printf '0' > "${counter_file}" + transient_command() { + local current_attempt + current_attempt="$(cat "${counter_file}")" + current_attempt=$((current_attempt + 1)) + printf '%s' "${current_attempt}" > "${counter_file}" + [ "${current_attempt}" -ge 3 ] + } + + local wait_seconds=0 + local -a cmd=(transient_command) + retry cmd wait_seconds 5 + attempts="$(cat "${counter_file}")" +} + +verify_invalid_role_example() { + local role="invalid" + verify_role role +} + +It 'writes and appends file contents' +When call write_file_example +The status should be success +The output should include "first" +The output should include "second" +End + +It 'retries commands until they succeed' +When call retry_until_success_example +The status should be success +The output should include "attempt #3 - " +The variable attempts should eq 3 +End + +It 'fails on unknown VMSS roles' +When run verify_invalid_role_example +The status should be failure +The output should include 'failed to verify role' +The output should include 'invalid' +End +End + +Describe 'util-packages.sh' +configure_repos_example() { + ensure_test_workspace + curl() { :; } + retry() { + local -n wait_ref="$2" + printf 'cmd=%s\nwait=%s\nretries=%s\n' "${cmd[*]}" "${wait_ref}" "${3:-}" + } + + local wait_time=7 + configure_rpm_repos wait_time 9 +} + +install_packages_example() { + ensure_test_workspace + retry() { + local -n wait_ref="$2" + printf 'cmd=%s\nwait=%s\nretries=%s\n' "${cmd[*]}" "${wait_ref}" "${3:-}" + } + + local wait_time=5 + local -a packages=(azure-cli podman) + dnf_install_pkgs packages wait_time 11 +} + +It 'configures the mariner extended repo through retry' +When call configure_repos_example +The status should be success +The output should include 'dnf update -y --enablerepo=cbl-mariner2.0prodextendedx86_64' +The output should include 'wait=7' +The output should include 'retries=9' +End + +It 'builds dnf install commands from package arrays' +When call install_packages_example +The status should be success +The output should include 'dnf -y install azure-cli' +The output should include 'podman' +The output should include 'wait=5' +The output should include 'retries=11' +End +End + +Describe 'util-system.sh' +configure_sshd_example() { + ensure_test_workspace + write_file_contents /etc/ssh/sshd_config 'PasswordAuthentication no' + systemctl() { printf '%s\n' "$*" >> "${CALL_LOG}"; } + + configure_sshd + + cat /etc/ssh/sshd_config +} + +configure_devproxy_certs_example() { + ensure_test_workspace + configure_certs_devproxy + printf '%s\n' "$(cat /etc/proxy/proxy.crt)" + printf '%s\n' "$(cat /etc/proxy/proxy.key)" + printf '%s\n' "$(cat /etc/proxy/proxy-client.crt)" + stat -c '%a' /etc/proxy/proxy.key +} + +It 'updates sshd config and reloads sshd' +When call configure_sshd_example +The status should be success +The output should include 'PasswordAuthentication yes' +The contents of file "${CALL_LOG}" should include 'reload sshd.service' +End + +It 'writes devproxy certificates with locked-down key permissions' +When call configure_devproxy_certs_example +The status should be success +The output should include 'cert' +The output should include 'key' +The output should include 'client' +The output should end with '600' +The path /etc/proxy/proxy.key should be file +End +End + +Describe 'util-services.sh' +enable_services_example() { + ensure_test_workspace + systemctl() { printf '%s\n' "$*" >> "${CALL_LOG}"; } + + local -a services=(aro-rp fluentbit) + enable_services services + + cat "${CALL_LOG}" +} + +configure_rp_service_example() { + ensure_test_workspace + local service_image="${RPIMAGE}" + local service_role="rp" + local service_conf="CUSTOM='value'" + local service_ip="10.88.0.2" + + configure_service_aro_rp service_image service_role service_conf service_ip + + cat /etc/sysconfig/aro-rp + printf '\n--service--\n' + cat /etc/systemd/system/aro-rp.service +} + +configure_gateway_service_example() { + ensure_test_workspace + local service_image="${RPIMAGE}" + local service_role="gateway" + local service_conf="GATEWAY='enabled'" + local service_ip="10.88.0.3" + + configure_service_aro_gateway service_image service_role service_conf service_ip + + cat /etc/sysconfig/aro-gateway + printf '\n--service--\n' + cat /etc/systemd/system/aro-gateway.service +} + +It 'reloads systemd and enables requested services' +When call enable_services_example +The status should be success +The output should include 'daemon-reload' +The output should include 'enable --now aro-rp' +The output should include 'enable --now fluentbit' +End + +It 'writes the aro-rp systemd unit and env file' +When call configure_rp_service_example +The status should be success +The output should include "IPADDRESS='10.88.0.2'" +The output should include "ARO_LOG_LEVEL='debug'" +The output should include 'ExecStart=/usr/bin/podman run' +The output should include '-p 443:8443' +End + +It 'writes the aro-gateway systemd unit and env file' +When call configure_gateway_service_example +The status should be success +The output should include "IPADDRESS='10.88.0.3'" +The output should include "ROLE='gateway'" +The output should include 'ExecStart=/usr/bin/podman run' +The output should include '-p 80:8080' +The output should include '-p 443:8443' +End +End +End