k3s/scripts/sonobuoy

#!/bin/bash

set -xe

cd $(dirname $0)/..

if [ -z "${K3S_IMAGE}" ]; then
    echo 'K3S_IMAGE environment variable should be defined'
    exit 1
fi

# ---

port-used() {
    (cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
}
export -f port-used

# ---

get-port() {
    while
        PORT=$((10000 + RANDOM % 50000))
        port-used ${PORT}
    do continue; done
    echo ${PORT}
}
export -f get-port

# ---

fetch-kubeconfig() {
    docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
}
export -f fetch-kubeconfig

# ---

wait-for-kubeconfig() {
    while ! fetch-kubeconfig; do
        echo 'Waiting for kubeconfig to become available...'
        sleep 5
    done
}
export -f wait-for-kubeconfig

# ---

count-ready-nodes() {
    kubectl get nodes -o json \
        | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
        | wc -l \
        | tr -d '[:space:]'
}
export -f count-ready-nodes

# ---

wait-for-nodes() {
    while [[ $(count-ready-nodes) -ne $1 ]]; do
        echo 'Waiting for nodes to be ready...'
        sleep 5
    done
}
export -f wait-for-nodes

# ---

pod-ready() {
    kubectl get pods -n kube-system -o json \
        | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
}
export -f pod-ready

# ---

wait-for-services() {
    for service in $@; do
        while [[ "$(pod-ready ${service})" != 'true' ]]; do
            echo "Waiting for service ${service} to be ready..."
            sleep 5
        done
        echo "Service ${service} is ready"
    done
}
export -f wait-for-services

# ---

verify-valid-version() {
    if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
        return 1
    fi
}
export -f verify-valid-version

# ---

verify-valid-versions() {
    verify-valid-version $1 kubectl version
    verify-valid-version $1 ctr version
    verify-valid-version $1 crictl version
}
export -f verify-valid-versions

# ---

dump-container-logs() {
    mkdir -p ${LOGS}
    for container in ${CONTAINERS}; do
        docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/containerd-${container}.log
        docker logs ${container} >${LOGS}/${container}.log 2>&1
    done
}
export -f dump-container-logs

# ---

retrieve-sonobuoy-logs() {
    if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
        status=passed
        exit_code=0
    else
        status=failed
        exit_code=1
    fi

    mkdir -p ${E2E}
    sonobuoy retrieve ${E2E}
    tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
    if [ ! -s ${RESULTS} ]; then
        return 1
    fi
    if [ -n "${E2E_LOG_OUTPUT}" ]; then
        cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
    fi
    awk '/^Summarizing .* Failures:$/,0' ${RESULTS}
    return ${exit_code}
}
export -f retrieve-sonobuoy-logs

# ---

sonobuoy-test() {
    time sonobuoy run \
        --config=scripts/sonobuoy-config.json \
        --plugin-env=e2e.E2E_USE_GO_RUNNER=true \
        --wait=30 \
        "${@}" &
    SONOBUOY_PID=$!
    wait $SONOBUOY_PID
    retrieve-sonobuoy-logs
}
export -f sonobuoy-test

# ---

cleanup() {
    exit_status=$?
    set +e
    echo 'Cleaning up'
    trap - EXIT
    [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null
    if [ "${exit_status}" -ne "0" ]; then
        dump-container-logs
    fi
    docker rm -f ${CONTAINERS} 2>/dev/null
    rm ${KUBECONFIG}
    exit ${exit_status}
}
trap cleanup EXIT

# ---

K3S_PORT=$(timeout --foreground 5s bash -c get-port)
OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
LOGS=$(pwd)/logs/$$
E2E="${OUTPUT}/e2e"
E2E_LOG='plugins/e2e/results/global/e2e.log'
RESULTS="${E2E}/${E2E_LOG}"

SECRET=random-$((100000 + RANDOM % 999999))
export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
export KUBECONFIG=${OUTPUT}/kubeconfig.yaml

mkdir -p ${OUTPUT}

# ---

docker run -d --name ${K3S_SERVER} --privileged \
    -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
    -e K3S_CLUSTER_SECRET=${SECRET} \
    ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}

K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}"

timeout --foreground 1m bash -c wait-for-kubeconfig
verify-valid-versions ${K3S_SERVER}

# ---

docker run -d --name ${K3S_AGENT} --privileged \
    -e K3S_CLUSTER_SECRET=${SECRET} \
    -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \
    ${K3S_IMAGE} agent

echo "Started ${K3S_AGENT}"

# ---

timeout --foreground 1m bash -c 'wait-for-nodes 2'
timeout --foreground 3m bash -c 'wait-for-services coredns local-path-provisioner metrics-server'

if [ "$ARCH" = 'arm' ]; then
    echo "Aborting sonobuoy tests, images not available for $ARCH"
    exit 0
fi

echo 'Starting sonobuoy tests'
sonobuoy-test "${@}"