Clean up sonobuoy scripts

scripts/sonobuoy

@@ -28,6 +28,7 @@ export -f get-port
 K3S_PORT=$(timeout --foreground 5s bash -c get-port)
 OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
 mkdir -p ${OUTPUT}
+RESULTS="${OUTPUT}/plugins/e2e/results/e2e.log"
 
 SECRET=random-$((100000 + RANDOM % 999999))
 export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
@@ -49,17 +50,6 @@ trap cleanup EXIT
 
 # ---
 
-dump-logs() {
-    for container in ${CONTAINERS}; do
-        docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${OUTPUT}/${container}-containerd.log
-        docker logs ${container} >${OUTPUT}/${container}.log 2>&1
-        cat ${OUTPUT}/${container}-containerd.log
-        cat ${OUTPUT}/${container}.log
-    done
-}
-
-# ---
-
 docker run -d --name ${K3S_SERVER} --privileged \
     -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
     -e K3S_CLUSTER_SECRET=${SECRET} \
@@ -137,32 +127,76 @@ timeout --foreground 1m bash -c wait-for-services
 
 # ---
 
-echo "Starting sonobuoy tests"
+dump-container-logs() {
+    for container in ${CONTAINERS}; do
+        docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${OUTPUT}/${container}-containerd.log
+        docker logs ${container} >${OUTPUT}/${container}.log 2>&1
+        cat ${OUTPUT}/${container}-containerd.log
+        cat ${OUTPUT}/${container}.log
+    done
+}
+export -f dump-container-logs
 
-timeout --foreground 30m sonobuoy run \
-    --config scripts/sonobuoy-config.json \
-    --wait \
-    ${@}
-sonobuoy status
-sonobuoy retrieve ${OUTPUT}
+# ---
 
-(
-    cd ${OUTPUT}
-    tar xzf *_sonobuoy_*.tar.gz
-    results="./plugins/e2e/results/e2e.log"
-    [ -s ${results} ] || {
-        dump-logs
-        exit 1
-    }
-    tail -20 ${results}
+sonobuoy-destruct() {
+    sonobuoy logs
+    sonobuoy delete --wait
+}
+export -f sonobuoy-destruct
 
-    exit_code=0
-    status=$(tail -5 ${results} | grep '^SUCCESS!.*| 0 Failed |' >/dev/null && echo passed || echo failed)
-    [ "${status}" = "failed" ] && exit_code=1
+# ---
 
-    if [ -n "${E2E_LOG_OUTPUT}" ]; then
-        cp ${results} $(echo ${E2E_LOG_OUTPUT} | sed -e "s/-STATUS-/-${status}-/g")
+sonobuoy-test() {
+    timeout --foreground 30m sonobuoy run \
+        --config scripts/sonobuoy-config.json \
+        --wait \
+        "${@}"
+    if sonobuoy status | grep "failed"; then
+        sonobuoy-destruct
+        return 1
+    fi
+    sonobuoy retrieve ${OUTPUT}
+
+    tar x -z -f ${OUTPUT}/*_sonobuoy_*.tar.gz -C ${OUTPUT}
+    if [ ! -s ${RESULTS} ]; then
+        sonobuoy-destruct
+        return 1
     fi
 
-    exit ${exit_code}
-)
+    tail -20 ${RESULTS}
+}
+export -f sonobuoy-test
+
+# ---
+
+sonobuoy-retry-test() {
+    SECONDS=0
+    LIMIT=300
+    for i in seq 1 3; do
+        sonobuoy-test "${@}" && return
+        echo "Failed sonobuoy try #${i}"
+        if [[ "$SECONDS" > "$LIMIT" ]]; then
+            echo "Stopping sonobuoy re-try after ${SECONDS} seconds (limit ${LIMIT}s)"
+            break
+        fi
+    done
+    dump-container-logs
+    return 1
+}
+export -f sonobuoy-retry-test
+
+# ---
+
+echo "Starting sonobuoy tests"
+sonobuoy-retry-test "${@}"
+
+exit_code=0
+status=$(tail -5 ${RESULTS} | grep '^SUCCESS!.*| 0 Failed |' >/dev/null && echo passed || echo failed)
+[ "${status}" = "failed" ] && exit_code=1
+
+if [ -n "${E2E_LOG_OUTPUT}" ]; then
+    cp ${RESULTS} $(echo ${E2E_LOG_OUTPUT} | sed -e "s/-STATUS-/-${status}-/g")
+fi
+
+exit ${exit_code}

scripts/sonobuoy-e2e-tests

@@ -6,8 +6,8 @@ cd $(dirname $0)/..
 cleanup() {
     exit_code=$?
     set +e
-    echo "Waiting on the tests!"
     wait
+    echo "Finished the tests!"
     exit ${exit_code}
 }
 trap cleanup EXIT
@@ -40,9 +40,6 @@ log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log \
     run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y
 
 
-exit_code=0
 for pid in "${pids[@]}"; do
-    wait $pid || exit_code=$?
+    wait $pid || exit $?
 done
-trap - EXIT
-exit ${exit_code}