k3s/scripts/sonobuoy

222 lines
4.7 KiB
Plaintext
Raw Normal View History

2019-04-29 20:54:51 +00:00
#!/bin/bash
set -xe
cd $(dirname $0)/..
if [ -z "${K3S_IMAGE}" ]; then
echo "K3S_IMAGE environment variable should be defined"
exit 1
fi
# ---
port-used() {
(cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
}
export -f port-used
2019-07-04 15:07:26 +00:00
# ---
2019-04-29 20:54:51 +00:00
get-port() {
while
PORT=$((10000 + RANDOM % 50000))
port-used ${PORT}
do continue; done
echo ${PORT}
}
export -f get-port
# ---
fetch-kubeconfig() {
2019-05-03 18:35:58 +00:00
docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
2019-04-29 20:54:51 +00:00
}
export -f fetch-kubeconfig
2019-07-04 15:07:26 +00:00
# ---
2019-04-29 20:54:51 +00:00
wait-for-kubeconfig() {
2019-05-03 18:35:58 +00:00
while ! fetch-kubeconfig; do
2019-04-29 20:54:51 +00:00
echo "Waiting for kubeconfig to become available..."
sleep 5
done
}
export -f wait-for-kubeconfig
2019-05-03 18:35:58 +00:00
# ---
2019-04-29 20:54:51 +00:00
count-ready-nodes() {
kubectl get nodes -o json \
| jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
| wc -l \
| tr -d '[:space:]'
}
export -f count-ready-nodes
2019-07-04 15:07:26 +00:00
# ---
2019-04-29 20:54:51 +00:00
wait-for-nodes() {
while [[ "$(count-ready-nodes)" != "2" ]]; do
echo "Waiting for nodes to be ready..."
sleep 5
done
}
export -f wait-for-nodes
# ---
pod-ready() {
kubectl get pods -n kube-system -o json \
| jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
}
export -f pod-ready
2019-07-04 15:07:26 +00:00
# ---
2019-04-29 20:54:51 +00:00
wait-for-services() {
for service in coredns; do
while [[ "$(pod-ready ${service})" != "true" ]]; do
echo "Waiting for service ${service} to be ready..."
sleep 5
done
echo "Service ${service} is ready"
done
}
export -f wait-for-services
# ---
2019-07-02 17:07:26 +00:00
dump-container-logs() {
2019-07-28 00:01:10 +00:00
mkdir -p ${LOGS}
2019-07-02 17:07:26 +00:00
for container in ${CONTAINERS}; do
2019-07-28 00:01:10 +00:00
docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/containerd-${container}.log
docker logs ${container} >${LOGS}/${container}.log 2>&1
2019-07-02 17:07:26 +00:00
done
}
export -f dump-container-logs
# ---
2019-04-29 20:54:51 +00:00
2019-07-02 17:07:26 +00:00
sonobuoy-destruct() {
2019-07-28 00:01:10 +00:00
mkdir -p ${LOGS}
sonobuoy logs >${LOGS}/$i-sonobuoy.log 2>&1
2019-07-02 17:07:26 +00:00
sonobuoy delete --wait
}
export -f sonobuoy-destruct
# ---
sonobuoy-test() {
timeout --foreground 30m sonobuoy run \
--config scripts/sonobuoy-config.json \
--wait \
"${@}"
if sonobuoy status | grep "failed"; then
sonobuoy-destruct
return 1
fi
2019-07-04 15:07:26 +00:00
rm -rf ${E2E}
mkdir -p ${E2E}
sonobuoy retrieve ${E2E}
tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
2019-07-02 17:07:26 +00:00
if [ ! -s ${RESULTS} ]; then
sonobuoy-destruct
return 1
2019-04-29 20:54:51 +00:00
fi
2019-07-01 15:29:57 +00:00
2019-07-02 17:07:26 +00:00
tail -20 ${RESULTS}
}
export -f sonobuoy-test
# ---
sonobuoy-retry-test() {
SECONDS=0
LIMIT=300
2019-07-02 22:11:39 +00:00
for i in $(seq 1 3); do
2019-07-02 17:07:26 +00:00
sonobuoy-test "${@}" && return
2019-07-04 15:07:26 +00:00
echo "*** Failed Sonobuoy try #${i} on port ${K3S_PORT} at ${SECONDS} seconds ***"
2019-07-02 22:11:39 +00:00
if [ "$SECONDS" -gt "$LIMIT" ]; then
2019-07-02 17:07:26 +00:00
echo "Stopping sonobuoy re-try after ${SECONDS} seconds (limit ${LIMIT}s)"
break
fi
done
return 1
}
export -f sonobuoy-retry-test
# ---
2019-07-04 15:07:26 +00:00
cleanup() {
exit_status=$?
set +e
echo "Cleaning up"
2019-07-26 22:10:39 +00:00
if [ "${exit_status}" -ne "0" ]; then
dump-container-logs
fi
2019-07-04 15:07:26 +00:00
docker rm -f ${CONTAINERS} 2>/dev/null
rm ${KUBECONFIG}
exit ${exit_status}
}
trap cleanup EXIT
# ---
K3S_PORT=$(timeout --foreground 5s bash -c get-port)
OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
2019-07-28 00:01:10 +00:00
LOGS=$(pwd)/logs/$$
2019-07-04 15:07:26 +00:00
E2E="${OUTPUT}/e2e"
E2E_LOG="plugins/e2e/results/e2e.log"
RESULTS="${E2E}/${E2E_LOG}"
mkdir -p ${OUTPUT}
SECRET=random-$((100000 + RANDOM % 999999))
export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
# ---
docker run -d --name ${K3S_SERVER} --privileged \
-p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
-e K3S_CLUSTER_SECRET=${SECRET} \
${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}
K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}"
# ---
timeout --foreground 1m bash -c wait-for-kubeconfig
docker run -d --name ${K3S_AGENT} --privileged \
-e K3S_CLUSTER_SECRET=${SECRET} \
-e K3S_URL=https://${K3S_IP}:${K3S_PORT} \
${K3S_IMAGE} agent
echo "Started ${K3S_AGENT}"
# ---
timeout --foreground 1m bash -c wait-for-nodes
timeout --foreground 1m bash -c wait-for-services
2019-07-02 17:07:26 +00:00
echo "Starting sonobuoy tests"
sonobuoy-retry-test "${@}"
2019-07-04 15:07:26 +00:00
# ---
2019-07-02 17:07:26 +00:00
exit_code=0
status=$(tail -5 ${RESULTS} | grep '^SUCCESS!.*| 0 Failed |' >/dev/null && echo passed || echo failed)
[ "${status}" = "failed" ] && exit_code=1
if [ -n "${E2E_LOG_OUTPUT}" ]; then
cp ${RESULTS} $(echo ${E2E_LOG_OUTPUT} | sed -e "s/-STATUS-/-${status}-/g")
fi
exit ${exit_code}