#!/bin/bash set -xe cd $(dirname $0)/.. if [ -z "${K3S_IMAGE}" ]; then echo "K3S_IMAGE environment variable should be defined" exit 1 fi # --- port-used() { (cat /dev/tcp/127.0.0.1/$1) 2>/dev/null } export -f port-used # --- get-port() { while PORT=$((10000 + RANDOM % 50000)) port-used ${PORT} do continue; done echo ${PORT} } export -f get-port # --- fetch-kubeconfig() { docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null } export -f fetch-kubeconfig # --- wait-for-kubeconfig() { while ! fetch-kubeconfig; do echo "Waiting for kubeconfig to become available..." sleep 5 done } export -f wait-for-kubeconfig # --- count-ready-nodes() { kubectl get nodes -o json \ | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \ | wc -l \ | tr -d '[:space:]' } export -f count-ready-nodes # --- wait-for-nodes() { while [[ "$(count-ready-nodes)" != "2" ]]; do echo "Waiting for nodes to be ready..." sleep 5 done } export -f wait-for-nodes # --- pod-ready() { kubectl get pods -n kube-system -o json \ | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null } export -f pod-ready # --- wait-for-services() { for service in coredns; do while [[ "$(pod-ready ${service})" != "true" ]]; do echo "Waiting for service ${service} to be ready..." sleep 5 done echo "Service ${service} is ready" done } export -f wait-for-services # --- dump-container-logs() { mkdir -p ${LOGS} for container in ${CONTAINERS}; do docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/containerd-${container}.log docker logs ${container} >${LOGS}/${container}.log 2>&1 done } export -f dump-container-logs # --- sonobuoy-destruct() { mkdir -p ${LOGS} sonobuoy logs >${LOGS}/$i-sonobuoy.log 2>&1 sonobuoy delete --wait } export -f sonobuoy-destruct # --- sonobuoy-test() { timeout --foreground 30m sonobuoy run \ --config scripts/sonobuoy-config.json \ --wait \ "${@}" if sonobuoy status | grep "failed"; then sonobuoy-destruct return 1 fi rm -rf ${E2E} mkdir -p ${E2E} sonobuoy retrieve ${E2E} tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG} if [ ! -s ${RESULTS} ]; then sonobuoy-destruct return 1 fi tail -20 ${RESULTS} } export -f sonobuoy-test # --- sonobuoy-retry-test() { SECONDS=0 LIMIT=300 for i in $(seq 1 3); do sonobuoy-test "${@}" && return echo "*** Failed Sonobuoy try #${i} on port ${K3S_PORT} at ${SECONDS} seconds ***" if [ "$SECONDS" -gt "$LIMIT" ]; then echo "Stopping sonobuoy re-try after ${SECONDS} seconds (limit ${LIMIT}s)" break fi done return 1 } export -f sonobuoy-retry-test # --- cleanup() { exit_status=$? set +e echo "Cleaning up" if [ "${exit_status}" -ne "0" ]; then dump-container-logs fi docker rm -f ${CONTAINERS} 2>/dev/null rm ${KUBECONFIG} exit ${exit_status} } trap cleanup EXIT # --- K3S_PORT=$(timeout --foreground 5s bash -c get-port) OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT} LOGS=$(pwd)/logs/$$ E2E="${OUTPUT}/e2e" E2E_LOG="plugins/e2e/results/global/e2e.log" RESULTS="${E2E}/${E2E_LOG}" mkdir -p ${OUTPUT} SECRET=random-$((100000 + RANDOM % 999999)) export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT} export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT} export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}" export KUBECONFIG=${OUTPUT}/kubeconfig.yaml # --- docker run -d --name ${K3S_SERVER} --privileged \ -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \ -e K3S_CLUSTER_SECRET=${SECRET} \ ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT} K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER}) echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}" # --- timeout --foreground 1m bash -c wait-for-kubeconfig docker run -d --name ${K3S_AGENT} --privileged \ -e K3S_CLUSTER_SECRET=${SECRET} \ -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \ ${K3S_IMAGE} agent echo "Started ${K3S_AGENT}" # --- timeout --foreground 1m bash -c wait-for-nodes timeout --foreground 1m bash -c wait-for-services echo "Starting sonobuoy tests" sonobuoy-retry-test "${@}" # --- exit_code=0 status=$(tail -5 ${RESULTS} | grep '^SUCCESS!.*| 0 Failed |' >/dev/null && echo passed || echo failed) [ "${status}" = "failed" ] && exit_code=1 if [ -n "${E2E_LOG_OUTPUT}" ]; then cp ${RESULTS} $(echo ${E2E_LOG_OUTPUT} | sed -e "s/-STATUS-/-${status}-/g") fi exit ${exit_code}