diff --git a/scripts/sonobuoy b/scripts/sonobuoy index edf2a17fb7..97c4b27b37 100755 --- a/scripts/sonobuoy +++ b/scripts/sonobuoy @@ -98,63 +98,51 @@ export -f dump-container-logs # --- -sonobuoy-destruct() { - mkdir -p ${LOGS} - sonobuoy logs >${LOGS}/$i-sonobuoy.log 2>&1 - sonobuoy delete --wait +retrieve-sonobuoy-logs() { + if sonobuoy status | grep -q -E " +e2e +complete +passed +"; then + status=passed + exit_code=0 + else + status=failed + exit_code=1 + fi + + mkdir -p ${E2E} + sonobuoy retrieve ${E2E} + tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG} + if [ ! -s ${RESULTS} ]; then + return 1 + fi + if [ -n "${E2E_LOG_OUTPUT}" ]; then + cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}") + fi + awk '/^Summarizing .* Failures:$/,0' ${RESULTS} + return ${exit_code} } -export -f sonobuoy-destruct +export -f retrieve-sonobuoy-logs # --- sonobuoy-test() { - timeout --foreground 30m sonobuoy run \ - --config scripts/sonobuoy-config.json \ + time sonobuoy run \ + --config=scripts/sonobuoy-config.json \ --plugin-env=e2e.E2E_USE_GO_RUNNER=true \ - --wait \ - "${@}" - if sonobuoy status | grep "failed"; then - sonobuoy-destruct - return 1 - fi - - rm -rf ${E2E} - mkdir -p ${E2E} - sonobuoy retrieve ${E2E} - - tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG} - if [ ! -s ${RESULTS} ]; then - sonobuoy-destruct - return 1 - fi - - tail -20 ${RESULTS} + --wait=30 \ + "${@}" & + SONOBUOY_PID=$! + wait $SONOBUOY_PID + retrieve-sonobuoy-logs } export -f sonobuoy-test # --- -sonobuoy-retry-test() { - SECONDS=0 - LIMIT=300 - for i in $(seq 1 3); do - sonobuoy-test "${@}" && return - echo "*** Failed Sonobuoy try #${i} on port ${K3S_PORT} at ${SECONDS} seconds ***" - if [ "$SECONDS" -gt "$LIMIT" ]; then - echo "Stopping sonobuoy re-try after ${SECONDS} seconds (limit ${LIMIT}s)" - break - fi - done - return 1 -} -export -f sonobuoy-retry-test - -# --- - cleanup() { exit_status=$? set +e echo "Cleaning up" + trap - EXIT + [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null if [ "${exit_status}" -ne "0" ]; then dump-container-logs fi @@ -172,7 +160,6 @@ LOGS=$(pwd)/logs/$$ E2E="${OUTPUT}/e2e" E2E_LOG="plugins/e2e/results/global/e2e.log" RESULTS="${E2E}/${E2E_LOG}" -mkdir -p ${OUTPUT} SECRET=random-$((100000 + RANDOM % 999999)) export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT} @@ -180,6 +167,8 @@ export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT} export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}" export KUBECONFIG=${OUTPUT}/kubeconfig.yaml +mkdir -p ${OUTPUT} + # --- docker run -d --name ${K3S_SERVER} --privileged \ @@ -207,16 +196,4 @@ timeout --foreground 1m bash -c wait-for-nodes timeout --foreground 1m bash -c wait-for-services echo "Starting sonobuoy tests" -sonobuoy-retry-test "${@}" - -# --- - -exit_code=0 -status=$(tail -5 ${RESULTS} | grep '^SUCCESS!.*| 0 Failed |' >/dev/null && echo passed || echo failed) -[ "${status}" = "failed" ] && exit_code=1 - -if [ -n "${E2E_LOG_OUTPUT}" ]; then - cp ${RESULTS} $(echo ${E2E_LOG_OUTPUT} | sed -e "s/-STATUS-/-${status}-/g") -fi - -exit ${exit_code} +sonobuoy-test "${@}" diff --git a/scripts/sonobuoy-e2e-tests b/scripts/sonobuoy-e2e-tests index fd178e0673..5952b8b5b5 100755 --- a/scripts/sonobuoy-e2e-tests +++ b/scripts/sonobuoy-e2e-tests @@ -17,6 +17,23 @@ mkdir -p ${OUTPUT} pids=() output=() +cleanup() { + exit_status=$? + set +e + trap - EXIT INT + kill ${pids[@]} 2>/dev/null + wait + set +x + echo -n "Tests " + if [ "${exit_status}" -eq "0" ]; then + echo "passed" + else + echo "failed" + fi + exit ${exit_status} +} +trap cleanup EXIT INT + run-sonobuoy() { output+=(${log_output}) E2E_LOG_OUTPUT=${log_output} ./scripts/sonobuoy ${@} & @@ -31,40 +48,8 @@ sleep 60 log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log \ run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]' -show-logs() { - for pid in "${pids[@]}"; do - logdir=$(pwd)/logs/${pid} - if [ ! -d $logdir ]; then - continue - fi - echo - echo "#- Begin: logs for sonobuoy run pid ${pid}" - for log in $(pwd)/logs/${pid}/*; do - if [ -f ${log} ]; then - echo - echo "#- Start: ${log}" - cat ${log} - echo "#- End: ${log}" - echo - fi - done - echo "#- Finish: logs for sonobuoy run pid ${pid}" - echo - done -} - -cleanup() { - exit_status=$? - set +e +x - wait - echo "Finished the tests!" - if [ "${exit_status}" -ne "0" ]; then - show-logs - fi - exit ${exit_status} -} -trap cleanup EXIT - +exit_code=0 for pid in "${pids[@]}"; do - wait $pid || exit $? + wait $pid || exit_code=$? done +exit ${exit_code}