Merge pull request #898 from erikwilson/update-sonobuoy-builds

Clean up sonobuoy testing

scripts/sonobuoy

@@ -98,63 +98,51 @@ export -f dump-container-logs
 
 # ---
 
-sonobuoy-destruct() {
-    mkdir -p ${LOGS}
-    sonobuoy logs >${LOGS}/$i-sonobuoy.log 2>&1
-    sonobuoy delete --wait
+retrieve-sonobuoy-logs() {
+    if sonobuoy status | grep -q -E " +e2e +complete +passed +"; then
+        status=passed
+        exit_code=0
+    else
+        status=failed
+        exit_code=1
+    fi
+
+    mkdir -p ${E2E}
+    sonobuoy retrieve ${E2E}
+    tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
+    if [ ! -s ${RESULTS} ]; then
+        return 1
+    fi
+    if [ -n "${E2E_LOG_OUTPUT}" ]; then
+        cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
+    fi
+    awk '/^Summarizing .* Failures:$/,0' ${RESULTS}
+    return ${exit_code}
 }
-export -f sonobuoy-destruct
+export -f retrieve-sonobuoy-logs
 
 # ---
 
 sonobuoy-test() {
-    timeout --foreground 30m sonobuoy run \
-        --config scripts/sonobuoy-config.json \
+    time sonobuoy run \
+        --config=scripts/sonobuoy-config.json \
         --plugin-env=e2e.E2E_USE_GO_RUNNER=true \
-        --wait \
-        "${@}"
-    if sonobuoy status | grep "failed"; then
-        sonobuoy-destruct
-        return 1
-    fi
-
-    rm -rf ${E2E}
-    mkdir -p ${E2E}
-    sonobuoy retrieve ${E2E}
-
-    tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
-    if [ ! -s ${RESULTS} ]; then
-        sonobuoy-destruct
-        return 1
-    fi
-
-    tail -20 ${RESULTS}
+        --wait=30 \
+        "${@}" &
+    SONOBUOY_PID=$!
+    wait $SONOBUOY_PID
+    retrieve-sonobuoy-logs
 }
 export -f sonobuoy-test
 
 # ---
 
-sonobuoy-retry-test() {
-    SECONDS=0
-    LIMIT=300
-    for i in $(seq 1 3); do
-        sonobuoy-test "${@}" && return
-        echo "*** Failed Sonobuoy try #${i} on port ${K3S_PORT} at ${SECONDS} seconds ***"
-        if [ "$SECONDS" -gt "$LIMIT" ]; then
-            echo "Stopping sonobuoy re-try after ${SECONDS} seconds (limit ${LIMIT}s)"
-            break
-        fi
-    done
-    return 1
-}
-export -f sonobuoy-retry-test
-
-# ---
-
 cleanup() {
     exit_status=$?
     set +e
     echo "Cleaning up"
+    trap - EXIT
+    [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null
     if [ "${exit_status}" -ne "0" ]; then
         dump-container-logs
     fi
@@ -172,7 +160,6 @@ LOGS=$(pwd)/logs/$$
 E2E="${OUTPUT}/e2e"
 E2E_LOG="plugins/e2e/results/global/e2e.log"
 RESULTS="${E2E}/${E2E_LOG}"
-mkdir -p ${OUTPUT}
 
 SECRET=random-$((100000 + RANDOM % 999999))
 export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
@@ -180,6 +167,8 @@ export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
 export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
 export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
 
+mkdir -p ${OUTPUT}
+
 # ---
 
 docker run -d --name ${K3S_SERVER} --privileged \
@@ -207,16 +196,4 @@ timeout --foreground 1m bash -c wait-for-nodes
 timeout --foreground 1m bash -c wait-for-services
 
 echo "Starting sonobuoy tests"
-sonobuoy-retry-test "${@}"
-
-# ---
-
-exit_code=0
-status=$(tail -5 ${RESULTS} | grep '^SUCCESS!.*| 0 Failed |' >/dev/null && echo passed || echo failed)
-[ "${status}" = "failed" ] && exit_code=1
-
-if [ -n "${E2E_LOG_OUTPUT}" ]; then
-    cp ${RESULTS} $(echo ${E2E_LOG_OUTPUT} | sed -e "s/-STATUS-/-${status}-/g")
-fi
-
-exit ${exit_code}
+sonobuoy-test "${@}"

scripts/sonobuoy-e2e-tests

@@ -17,6 +17,23 @@ mkdir -p ${OUTPUT}
 pids=()
 output=()
 
+cleanup() {
+    exit_status=$?
+    set +e
+    trap - EXIT INT
+    kill ${pids[@]} 2>/dev/null
+    wait
+    set +x
+    echo -n "Tests "
+    if [ "${exit_status}" -eq "0" ]; then
+        echo "passed"
+    else
+        echo "failed"
+    fi
+    exit ${exit_status}
+}
+trap cleanup EXIT INT
+
 run-sonobuoy() {
     output+=(${log_output})
     E2E_LOG_OUTPUT=${log_output} ./scripts/sonobuoy ${@} &
@@ -31,40 +48,8 @@ sleep 60
 log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log \
     run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
 
-show-logs() {
-    for pid in "${pids[@]}"; do
-        logdir=$(pwd)/logs/${pid}
-        if [ ! -d $logdir ]; then
-            continue
-        fi
-        echo
-        echo "#- Begin: logs for sonobuoy run pid ${pid}"
-        for log in $(pwd)/logs/${pid}/*; do
-            if [ -f ${log} ]; then
-                echo
-                echo "#- Start: ${log}"
-                cat ${log}
-                echo "#- End: ${log}"
-                echo
-            fi
-        done
-        echo "#- Finish: logs for sonobuoy run pid ${pid}"
-        echo
-    done
-}
-
-cleanup() {
-    exit_status=$?
-    set +e +x
-    wait
-    echo "Finished the tests!"
-    if [ "${exit_status}" -ne "0" ]; then
-        show-logs
-    fi
-    exit ${exit_status}
-}
-trap cleanup EXIT
-
+exit_code=0
 for pid in "${pids[@]}"; do
-    wait $pid || exit $?
+    wait $pid || exit_code=$?
 done
+exit ${exit_code}