diff --git a/.drone.yml b/.drone.yml index 7f02ad3c75..a533a5c442 100644 --- a/.drone.yml +++ b/.drone.yml @@ -59,14 +59,14 @@ steps: event: - tag -- name: sonobuoy-e2e-tests +- name: test image: rancher/dapper:v0.4.2 secrets: [ gcloud_auth ] environment: GCLOUD_AUTH: from_secret: gcloud_auth commands: - - dapper -f Dockerfile.sonobuoy.dapper + - dapper -f Dockerfile.test.dapper volumes: - name: docker path: /var/run/docker.sock @@ -154,14 +154,14 @@ steps: event: - tag -- name: sonobuoy-e2e-tests +- name: test image: rancher/dapper:v0.4.2 secrets: [ gcloud_auth ] environment: GCLOUD_AUTH: from_secret: gcloud_auth commands: - - dapper -f Dockerfile.sonobuoy.dapper + - dapper -f Dockerfile.test.dapper volumes: - name: docker path: /var/run/docker.sock @@ -232,14 +232,14 @@ steps: event: - tag -- name: sonobuoy-e2e-tests +- name: test image: rancher/dapper:v0.4.2 secrets: [ gcloud_auth ] environment: GCLOUD_AUTH: from_secret: gcloud_auth commands: - - dapper -f Dockerfile.sonobuoy.dapper + - dapper -f Dockerfile.test.dapper volumes: - name: docker path: /var/run/docker.sock diff --git a/Dockerfile.sonobuoy.dapper b/Dockerfile.test.dapper similarity index 67% rename from Dockerfile.sonobuoy.dapper rename to Dockerfile.test.dapper index 2fab9379e5..367991d5a9 100644 --- a/Dockerfile.sonobuoy.dapper +++ b/Dockerfile.test.dapper @@ -1,22 +1,23 @@ FROM golang:1.13.5-alpine3.10 -RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2 -RUN SONOBUOY_VER=v0.16.2 && \ - SONOBUOY_PKG=github.com/vmware-tanzu/sonobuoy && \ - go get -d ${SONOBUOY_PKG} && \ - cd /go/src/${SONOBUOY_PKG} && \ - git checkout -b current ${SONOBUOY_VER} && \ - go build -o /usr/local/bin/sonobuoy -RUN rm -rf /go/src /go/pkg +RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2 openssl ARG DAPPER_HOST_ARCH ENV ARCH $DAPPER_HOST_ARCH +RUN if [ "${ARCH}" == "amd64" ] || [ "${ARCH}" == "arm64" ]; then \ + VERSION=0.17.0 OS=linux && \ + curl -sL "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_${ARCH}.tar.gz" | \ + tar -xzf - -C /usr/local/bin; \ + fi + RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \ curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt \ )/bin/linux/${ARCH}/kubectl -o /usr/local/bin/kubectl && \ chmod a+x /usr/local/bin/kubectl +ENV TEST_CLEANUP true + ENV DAPPER_RUN_ARGS --privileged --network host ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME GCLOUD_AUTH ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/ @@ -26,4 +27,4 @@ ENV HOME ${DAPPER_SOURCE} WORKDIR ${DAPPER_SOURCE} ENTRYPOINT ["./scripts/entry.sh"] -CMD ["sonobuoy-e2e-tests"] +CMD ["test"] diff --git a/scripts/log-upload b/scripts/log-upload index 1cb9762867..10589ac5f7 100755 --- a/scripts/log-upload +++ b/scripts/log-upload @@ -40,7 +40,7 @@ END HOME=${TMPDIR} PATH=$PATH:${HOME}/gsutil -LOG_TGZ=k3s-log-$(uname -s | tr '[:upper:]' '[:lower:]' )-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(date +%s)-$(basename $1).tgz +LOG_TGZ=k3s-log-$(date +%s)-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(basename $1).tgz tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1) gsutil cp ${TMPDIR}/${LOG_TGZ} gs://k3s-ci-logs || exit 1 diff --git a/scripts/sonobuoy b/scripts/sonobuoy deleted file mode 100755 index 182739f973..0000000000 --- a/scripts/sonobuoy +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -set -xe - -cd $(dirname $0)/.. - -. ./scripts/test-helpers - -if [ -z "${K3S_IMAGE}" ]; then - echo 'K3S_IMAGE environment variable should be defined' - exit 1 -fi - -# --- - -cleanup() { - exit_status=$? - set +e - echo 'Cleaning up' - trap - EXIT - [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null - if [ "${exit_status}" -ne "0" ]; then - dump-logs - fi - docker rm -f ${CONTAINERS} 2>/dev/null - rm ${KUBECONFIG} - exit ${exit_status} -} -trap cleanup EXIT - -# --- - -K3S_PORT=$(timeout --foreground 5s bash -c get-port) -OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT} -LOGS=$(pwd)/logs/$$ -E2E="${OUTPUT}/e2e" -E2E_LOG='plugins/e2e/results/global/e2e.log' -RESULTS="${E2E}/${E2E_LOG}" - -SECRET=random-$((100000 + RANDOM % 999999)) -export K3S_AGENT=k3s-agent-${K3S_PORT} -export K3S_SERVER=k3s-server-${K3S_PORT} -export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}" -export KUBECONFIG=${OUTPUT}/kubeconfig.yaml - -mkdir -p ${OUTPUT} - -# --- - -docker run -d --name ${K3S_SERVER} --privileged \ - -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \ - -e K3S_CLUSTER_SECRET=${SECRET} \ - -e K3S_DEBUG=true \ - ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT} - -K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER}) -echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}" - -docker exec ${K3S_SERVER} check-config || true -timeout --foreground 1m bash -c wait-for-kubeconfig -verify-valid-versions ${K3S_SERVER} - -# --- - -docker run -d --name ${K3S_AGENT} --privileged \ - -e K3S_CLUSTER_SECRET=${SECRET} \ - -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \ - ${K3S_IMAGE} agent - -echo "Started ${K3S_AGENT}" - -# --- - -timeout --foreground 1m bash -c 'wait-for-nodes 2' -timeout --foreground 3m bash -c 'wait-for-services coredns local-path-provisioner metrics-server' - -if [ "$ARCH" = 'arm' ]; then - echo "Aborting sonobuoy tests, images not available for $ARCH" - exit 0 -fi - -echo 'Starting sonobuoy tests' -sonobuoy-test "${@}" diff --git a/scripts/sonobuoy-e2e-tests b/scripts/sonobuoy-e2e-tests deleted file mode 100755 index 0f544eaf11..0000000000 --- a/scripts/sonobuoy-e2e-tests +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -set -e -x - -cd $(dirname $0)/.. - -if [ -z "$K3S_IMAGE" ]; then - . ./scripts/version.sh - TAG=${TAG:-${VERSION_TAG}${SUFFIX}} - REPO=${REPO:-rancher} - IMAGE_NAME=${IMAGE_NAME:-k3s} - export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG} -fi - -OUTPUT=$(pwd)/dist/artifacts -mkdir -p ${OUTPUT} - -pids=() -output=() - -show-logs() { - for pid in "${pids[@]}"; do - logdir=$(pwd)/logs/${pid} - if [ ! -d $logdir ]; then - continue - fi - echo - echo "#- Begin: logs for sonobuoy run pid ${pid}" - for log in $(pwd)/logs/${pid}/*.log; do - if [ -f ${log} ]; then - echo - echo "#- Tail: ${log}" - tail -10 ${log} - echo "#- Done: ${log}" - echo - fi - done - echo "#- Finish: logs for sonobuoy run pid ${pid}" - echo - done -} - -cleanup() { - exit_status=$? - set +e - trap - EXIT INT - kill ${pids[@]} 2>/dev/null - wait - set +x - echo -n "Tests " - if [ "${exit_status}" -eq "0" ]; then - echo "passed" - else - echo "failed" - show-logs - fi - exit ${exit_status} -} -trap cleanup EXIT INT - - -# --- - -run-sonobuoy() { - output+=(${log_output}) - E2E_LOG_OUTPUT=${log_output} \ - ./scripts/sonobuoy ${@} \ - > >(stdbuf -oL sed "s/^/[${label}] /") 2>&1 & - pids+=($!) -} - -log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log label=PARALLEL \ - run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y - -sleep 60 - -log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log label=SERIAL \ - run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]' - -exit_code=0 -for pid in "${pids[@]}"; do - wait $pid || exit_code=$? -done -exit ${exit_code} diff --git a/scripts/test b/scripts/test new file mode 100755 index 0000000000..f9e45c991b --- /dev/null +++ b/scripts/test @@ -0,0 +1,29 @@ +#!/bin/bash +set -e -x +cd $(dirname $0)/.. + +. ./scripts/test-helpers + +artifacts=$(pwd)/dist/artifacts +mkdir -p $artifacts + +# --- + +. ./scripts/test-run-basics + +# # --- + +E2E_OUTPUT=$artifacts test-run-sonobuoy + +# # --- + +# if [ "$ARCH" != 'amd64' ]; then +# printf "\033[33mSkipping remaining tests, images not available for $ARCH.\033[m\n" +# exit 0 +# fi + +# # --- + +# test-run-sonobuoy mysql +# test-run-sonobuoy postgres +# test-run-sonobuoy dqlite \ No newline at end of file diff --git a/scripts/test-certs-openssl.cnf b/scripts/test-certs-openssl.cnf new file mode 100644 index 0000000000..222288ec6d --- /dev/null +++ b/scripts/test-certs-openssl.cnf @@ -0,0 +1,72 @@ +# db OpenSSL configuration file. +SAN = "IP:127.0.0.1, IP:172.17.0.1, DNS:host.docker.internal" +dir = . + +[ ca ] +default_ca = db_ca + +[ db_ca ] +certs = $dir/certs +certificate = $dir/certs/ca.crt +crl = $dir/crl.pem +crl_dir = $dir/crl +crlnumber = $dir/crlnumber +database = $dir/index.txt +email_in_dn = no +new_certs_dir = $dir/newcerts +private_key = $dir/private/ca.key +serial = $dir/serial +RANDFILE = $dir/private/.rand +name_opt = ca_default +cert_opt = ca_default +default_days = 3650 +default_crl_days = 30 +default_md = sha512 +preserve = no +policy = policy_db + +[ policy_db ] +organizationName = optional +commonName = supplied + +[ req ] +default_bits = 1024 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +attributes = req_attributes +x509_extensions = v3_ca +string_mask = utf8only +req_extensions = db_client + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US +countryName_min = 2 +countryName_max = 2 +commonName = Common Name (FQDN) +0.organizationName = Organization Name (eg, company) +0.organizationName_default = db-ca + +[ req_attributes ] + +[ v3_ca ] +basicConstraints = CA:true +keyUsage = keyCertSign,cRLSign +subjectKeyIdentifier = hash + +[ db_client ] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth +keyUsage = digitalSignature, keyEncipherment + +[ db_peer ] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = digitalSignature, keyEncipherment +subjectAltName = ${ENV::SAN} + +[ db_server ] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = digitalSignature, keyEncipherment +subjectAltName = ${ENV::SAN} \ No newline at end of file diff --git a/scripts/test-helpers b/scripts/test-helpers index d425f31987..e30085b735 100755 --- a/scripts/test-helpers +++ b/scripts/test-helpers @@ -10,26 +10,31 @@ export -f port-used # --- get-port() { + local port=0 while - PORT=$((10000 + RANDOM % 50000)) - port-used ${PORT} + port=$((10000 + RANDOM % 50000)) + port-used $port do continue; done - echo ${PORT} + echo $port } export -f get-port # --- -fetch-kubeconfig() { - docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null -} +fetch-kubeconfig() {( + set -e -o pipefail + local num=${1:-1} + local name=$(cat $TEST_DIR/servers/$num/metadata/name) + local port=$(cat $TEST_DIR/servers/$num/metadata/port) + docker cp $name:/etc/rancher/k3s/k3s.yaml - 2>/dev/null | tar -xO 2>/dev/null | sed -e "s/:6443/:$port/g" >$TEST_DIR/servers/$num/kubeconfig.yaml +)} export -f fetch-kubeconfig # --- wait-for-kubeconfig() { - while ! fetch-kubeconfig; do - echo 'Waiting for kubeconfig to become available...' + while ! fetch-kubeconfig $1; do + echo 'Waiting for kubeconfig to become available...' >&2 sleep 5 done } @@ -49,7 +54,7 @@ export -f count-ready-nodes wait-for-nodes() { while [[ $(count-ready-nodes) -ne $1 ]]; do - echo 'Waiting for nodes to be ready...' + echo 'Waiting for nodes to be ready...' >&2 sleep 5 done } @@ -67,17 +72,31 @@ export -f pod-ready wait-for-services() { for service in $@; do - while [[ "$(pod-ready ${service})" != 'true' ]]; do - echo "Waiting for service ${service} to be ready..." + while [[ "$(pod-ready $service)" != 'true' ]]; do + echo "Waiting for service $service to be ready..." >&2 sleep 5 done - echo "Service ${service} is ready" + echo "Service $service is ready" done } export -f wait-for-services # --- +wait-for-db-connection() { + if [ -z "$DB_CONNECTION_TEST" ]; then + echo 'DB_CONNECTION_TEST is not defined' >&2 + return 1 + fi + while ! $DB_CONNECTION_TEST 2>/dev/null; do + echo 'Waiting for database to become available...' >&2 + sleep 5 + done +} +export -f wait-for-db-connection + +# --- + verify-valid-version() { if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then return 1 @@ -97,13 +116,30 @@ export -f verify-valid-versions # --- dump-logs() { - mkdir -p ${LOGS}/sonobuoy - tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${LOGS}/sonobuoy - for container in ${CONTAINERS}; do - docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/${container}-containerd.log - docker logs ${container} >${LOGS}/${container}-system.log 2>&1 + local testID=$(basename $TEST_DIR) + echo "#---------------------------------" + echo "#- Begin: logs for run ($testID)" + echo + for node in $TEST_DIR/*/*; do + [ -d "$node" ] || continue + local name=$(cat $node/metadata/name 2>/dev/null) + [ "$name" ] || continue + mkdir -p $node/logs + docker cp $name:/var/lib/rancher/k3s/agent/containerd/containerd.log $node/logs/containerd.log 2>/dev/null + docker logs $name >$node/logs/system.log 2>&1 + for log in $node/logs/*.log; do + echo + echo "#- Tail: $log" + tail -5 $log + echo "#- Done: $log" + echo + done done - ./scripts/log-upload ${LOGS} + echo + echo "#- Finish: logs for run ($testID)" + echo "#---------------------------------" + echo + ./scripts/log-upload $TEST_DIR } export -f dump-logs @@ -111,40 +147,389 @@ export -f dump-logs retrieve-sonobuoy-logs() { sonobuoy status || true - if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then - status=passed - exit_code=0 - else + + local status=passed + local code=0 + + if ! sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then status=failed - exit_code=1 + code=1 fi - mkdir -p ${E2E} - sonobuoy retrieve ${E2E} - tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG} - if [ ! -s ${RESULTS} ]; then + mkdir -p $TEST_DIR/sonobuoy + sonobuoy retrieve $TEST_DIR/sonobuoy 2>/dev/null || true + local logTarball=$TEST_DIR/sonobuoy/*_sonobuoy_*.tar.gz + + if [ -f $logTarball ]; then + tar -xz -f $logTarball -C $TEST_DIR/sonobuoy + rm $logTarball + else + rm -rf $TEST_DIR/sonobuoy + fi + + local e2eLog=$TEST_DIR/sonobuoy/plugins/e2e/results/global/e2e.log + if [ ! -s $e2eLog ]; then return 1 fi - if [ -n "${E2E_LOG_OUTPUT}" ]; then - cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}") + if [ -n "$LOG_OUTPUT" ]; then + cp $e2eLog $(sed -e "s/-STATUS-/-$status-/g" <<< "$LOG_OUTPUT") fi - awk '/^Summarizing .* Failures?:$/,0' ${RESULTS} - return ${exit_code} + awk '/^Summarizing .* Failures?:$/,0' $e2eLog + return $code } export -f retrieve-sonobuoy-logs # --- sonobuoy-test() { - time sonobuoy run \ - --config=scripts/sonobuoy-config.json \ - --plugin-env=e2e.E2E_USE_GO_RUNNER=true \ - --wait=30 \ - "${@}" & - SONOBUOY_PID=$! - wait $SONOBUOY_PID + if [ "$ARCH" = 'arm' ]; then + echo "Aborting sonobuoy tests, images not available for $ARCH" + return 0 + fi + echo 'Starting sonobuoy tests' + + sonobuoy run --config=scripts/sonobuoy-config.json --plugin-env=e2e.E2E_USE_GO_RUNNER=true --wait=30 $@ & + local sonobuoyPID=$! + local code=0 + time wait $sonobuoyPID || code=$? + retrieve-sonobuoy-logs + return $code } export -f sonobuoy-test # --- + +test-cleanup() { + local code=$? + set +e +x + echo 'Cleaning up...' + trap - EXIT INT TERM + if [[ $code -ne 0 ]]; then + dump-logs + fi + for name in $TEST_DIR/*/*/metadata/name; do + [ -f "$name" ] || continue + local container=$(cat $name) + docker rm -f -v $container >/dev/null 2>&1 & + done + if [ "$TEST_CLEANUP" = true ]; then + echo "Removing $TEST_DIR" + rm -rf $TEST_DIR >/dev/null 2>&1 & + fi + [ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK >/dev/null 2>&1 & + echo + echo -n "Test $(basename $TEST_DIR) " + if [ $code -eq 0 ]; then + echo "passed." + else + echo "failed." + fi + echo + exit $code +} +export -f test-cleanup + +# --- + +test-setup() { + export TEST_DIR=$(mktemp -d '/tmp/XXXXXX') + trap test-cleanup EXIT INT TERM + + mkdir -p $TEST_DIR/metadata + if [ "$LABEL" ]; then + exec > >(awk "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \ + 2> >(awk "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2) + echo "$LABEL" >$TEST_DIR/metadata/label + fi + + mkdir -p $TEST_DIR/logs + exec > >(tee -a $TEST_DIR/logs/test.log) \ + 2> >(tee -a $TEST_DIR/logs/test.log >&2) + + if [ -z "$K3S_IMAGE" ]; then + . ./scripts/version.sh + TAG=${TAG:-${VERSION_TAG}${SUFFIX}} + REPO=${REPO:-rancher} + IMAGE_NAME=${IMAGE_NAME:-k3s} + export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG} + fi + + if [ -z "$K3S_IMAGE" ]; then + echo 'K3S_IMAGE environment variable should be defined' + return 1 + fi + + local setupFile=./scripts/test-setup-${TEST_TYPE} + [ -f $setupFile ] && source $setupFile + + echo ${RANDOM}${RANDOM}${RANDOM} >$TEST_DIR/metadata/secret +} +export -f test-setup + +# --- + +gen-certs() {( + set -e -x + #umask 077 + + local opensslConfig=$(pwd)/scripts/test-certs-openssl.cnf + local subject="/C=US/ST=AZ/L=Tempe/O=Rancher/OU=DevOps/DC=com/DC=rancher" + local caDir=$TEST_DIR/db-ca + [ -d $caDir ] && rm -rf $caDir + mkdir -p $caDir + + cd $caDir + mkdir -p private certs newcerts crl + touch index.txt + echo '01' > serial + + openssl req \ + -config $opensslConfig \ + -new -x509 \ + -nodes \ + -subj $subject \ + -extensions v3_ca \ + -keyout private/ca.key \ + -out certs/ca.crt + + openssl req \ + -config $opensslConfig \ + -new \ + -nodes \ + -subj $subject"/CN=server" \ + -keyout private/db.key \ + -out db.csr + + openssl ca \ + -config $opensslConfig \ + -batch \ + -extensions db_server \ + -keyfile private/ca.key \ + -cert certs/ca.crt \ + -out certs/db.crt \ + -infiles db.csr + + openssl req \ + -config $opensslConfig \ + -new \ + -nodes \ + -subj $subject"/CN=client" \ + -keyout private/client.key \ + -out client.csr + + openssl ca \ + -config $opensslConfig \ + -batch \ + -extensions db_client \ + -keyfile private/ca.key \ + -cert certs/ca.crt \ + -out certs/client.crt \ + -infiles client.csr +)} +export -f gen-certs + +# --- + +inc-count() {( + shopt -s extglob + local count=$(exec 2>/dev/null; ls -1d $TEST_DIR/$1/+([0-9]) | xargs -n1 basename | sort -n -r | head -1) + count=$((count+1)) + mkdir -p $TEST_DIR/$1/$count/metadata + echo $count +)} +export -f inc-count + +# --- + +run-function() { + declare -f $1 >/dev/null 2>&1 || return 0 + $@ +} +export -f run-function + +# --- + +provision-server() { + local count=$(inc-count servers) + local testID=$(basename $TEST_DIR) + local name=$(echo "k3s-server-$count-$testID" | tee $TEST_DIR/servers/$count/metadata/name) + #local args=$(cat $TEST_DIR/args $TEST_DIR/servers/args $TEST_DIR/servers/$count/args 2>/dev/null) + local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/servers/$count/metadata/port) + local SERVER_INSTANCE_ARGS="SERVER_${count}_ARGS" + + run-function server-pre-hook $count + + docker run \ + -d --name $name \ + --privileged \ + -v $TEST_DIR/db-ca/:/db-ca \ + -p 127.0.0.1:$port:6443 \ + -p 6443 \ + -e K3S_TOKEN=$(cat $TEST_DIR/metadata/secret) \ + -e K3S_DEBUG=true \ + $K3S_IMAGE server $ARGS $SERVER_ARGS ${!SERVER_INSTANCE_ARGS} + + local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/servers/$count/metadata/ip) + local url=$(echo "https://$ip:6443" | tee $TEST_DIR/servers/$count/metadata/url) + + echo "Started $name @ $url" + run-function server-post-hook $count +} +export -f provision-server + +# --- + +provision-agent() { + local K3S_URL=${K3S_URL:-"$(cat $TEST_DIR/servers/1/metadata/url)"} + local count=$(inc-count agents) + local testID=$(basename $TEST_DIR) + local name=$(echo "k3s-agent-$count-$testID" | tee $TEST_DIR/agents/$count/metadata/name) + #local args=$(cat $TEST_DIR/args $TEST_DIR/agents/args $TEST_DIR/agents/$count/args 2>/dev/null) + local AGENT_INSTANCE_ARGS="AGENT_${count}_ARGS" + + run-function agent-pre-hook $count + docker run \ + -d --name $name \ + --privileged \ + -e K3S_TOKEN=$(cat $TEST_DIR/metadata/secret) \ + -e K3S_URL=$K3S_URL \ + $K3S_IMAGE agent $ARGS $AGENT_ARGS ${!AGENT_INSTANCE_ARGS} + + echo "Started $name" + run-function agent-post-hook $count +} +export -f provision-agent + +# --- + +provision-cluster() { + run-function cluster-pre-hook + + for i in $(seq 1 $NUM_SERVERS); do + provision-server + timeout --foreground 30s bash -c "wait-for-kubeconfig $i" + done + export KUBECONFIG=$TEST_DIR/servers/1/kubeconfig.yaml + + if [ $NUM_AGENTS -gt 0 ]; then + for _ in $(seq 1 $NUM_AGENTS); do + provision-agent + done + fi + + timeout --foreground 1m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))" + timeout --foreground 3m bash -c "wait-for-services $WAIT_SERVICES" + + run-function cluster-post-hook + + [ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK +} +export -f provision-cluster + +# --- + +run-test() { + export PROVISION_LOCK=$(mktemp) + ./scripts/test-runner $@ & + pids+=($!) + ( + set +x + while [ -f "$PROVISION_LOCK" ]; do + sleep 1 + done + sleep 5 + ) +} +export -f run-test + +# --- + +e2e-test() { + local label=$label + if [ -n "$LABEL_SUFFIX" ]; then + label="$label-$LABEL_SUFFIX" + fi + local logOutput= + if [ -n "$E2E_OUTPUT" ]; then + logOutput=$E2E_OUTPUT/$logname + fi + LABEL=$label LOG_OUTPUT=$logOutput run-test $@ +} + +# --- + +run-e2e-tests() { + label=PARALLEL \ + logName=e2e-STATUS-${ARCH}-parallel.log \ + e2e-test ${sonobuoyParallelArgs[@]} + + label=SERIAL \ + logName=e2e-STATUS-${ARCH}-serial.log \ + e2e-test ${sonobuoySerialArgs[@]} +} +export -f run-e2e-tests + +# --- + +test-run-sonobuoy() { + local suffix + if [ "$1" ]; then + suffix="-$1" + export LABEL_SUFFIX=$1 + fi + + . ./scripts/test-setup-sonobuoy$suffix + run-e2e-tests +} +export -f test-run-sonobuoy + +# --- + +pid-cleanup() { + local code=$? + set +e + local failCount=0 + if [ $code -ne 0 ]; then + for pid in ${pids[@]}; do + pkill -P $pid + wait $pid || failCount=$((failCount+1)) + done + fi + wait + trap - EXIT INT TERM + set +x + echo + if [ $failCount -eq 0 ]; then + printf '\033[32mAll tests passed.\033[m\n' + if [ $code -ne 0 ]; then + printf "\033[31mExit code is $code.\033[m\n" + fi + else + printf "\033[31m$failCount tests failed.\033[m\n" + fi + echo + exit $code +} +export -f pid-cleanup + +# --- + +wait-pids() { + trap - EXIT + set +e + local code=0 + for pid in "${pids[@]}"; do + wait $pid || code=$? + done + [ $code -eq 0 ] + pid-cleanup + exit $code +} +export -f wait-pids + +# --- + +pids=() +trap pid-cleanup INT TERM +trap wait-pids EXIT diff --git a/scripts/test-run-basics b/scripts/test-run-basics new file mode 100755 index 0000000000..02fcbe29c4 --- /dev/null +++ b/scripts/test-run-basics @@ -0,0 +1,14 @@ +#!/bin/bash + +export NUM_SERVERS=1 +export NUM_AGENTS=1 +export SERVER_ARGS='--no-deploy=traefik,coredns,local-storage,metrics-server' + +start-test() { + docker exec $(cat $TEST_DIR/servers/1/metadata/name) check-config || true + verify-valid-versions $(cat $TEST_DIR/servers/1/metadata/name) +} +export -f start-test + +# --- create a basic cluster and check for valid versions +LABEL=BASICS run-test diff --git a/scripts/test-runner b/scripts/test-runner new file mode 100755 index 0000000000..79b4ce10da --- /dev/null +++ b/scripts/test-runner @@ -0,0 +1,13 @@ +#!/bin/bash +set -x -e +cd $(dirname $0)/.. + +# --- + +for include in $TEST_INCLUDES; do + . $include +done + +test-setup +provision-cluster +start-test $@ diff --git a/scripts/test-setup-sonobuoy b/scripts/test-setup-sonobuoy new file mode 100755 index 0000000000..381d4d2067 --- /dev/null +++ b/scripts/test-setup-sonobuoy @@ -0,0 +1,14 @@ +#!/bin/bash + +export NUM_SERVERS=1 +export NUM_AGENTS=1 +export SERVER_ARGS='--no-deploy=traefik' +export WAIT_SERVICES='coredns local-path-provisioner metrics-server' + +export sonobuoyParallelArgs=(--e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y) +export sonobuoySerialArgs=(--e2e-focus='\[Serial\].*\[Conformance\]') + +start-test() { + sonobuoy-test $@ +} +export -f start-test diff --git a/scripts/test-setup-sonobuoy-dqlite b/scripts/test-setup-sonobuoy-dqlite new file mode 100644 index 0000000000..9b98b6e6c2 --- /dev/null +++ b/scripts/test-setup-sonobuoy-dqlite @@ -0,0 +1,18 @@ +#!/bin/bash + +. ./scripts/test-setup-sonobuoy + +export NUM_SERVERS=2 +export NUM_AGENTS=0 + +export SERVER_1_ARGS=--cluster-init + +# --- + +server-post-hook() { + if [ $1 -eq 1 ]; then + local url=$(cat $TEST_DIR/servers/1/metadata/url) + export SERVER_ARGS="--server $url" + fi +} +export -f server-post-hook diff --git a/scripts/test-setup-sonobuoy-mysql b/scripts/test-setup-sonobuoy-mysql new file mode 100644 index 0000000000..0ab361a207 --- /dev/null +++ b/scripts/test-setup-sonobuoy-mysql @@ -0,0 +1,56 @@ +#!/bin/bash + +. ./scripts/test-setup-sonobuoy + +# --- + +cluster-pre-hook() { + # gen-certs + + mkdir -p $TEST_DIR/db/$LABEL_SUFFIX/metadata + local testID=$(basename $TEST_DIR) + local name=$(echo $LABEL_SUFFIX-$testID | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/name) + local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/port) + local secret=$(echo "${RANDOM}${RANDOM}${RANDOM}" | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/secret) + + docker run --name $name \ + --privileged \ + -p 0.0.0.0:$port:3306 \ + -v $TEST_DIR/db-ca/:/db-ca \ + -e MYSQL_ROOT_PASSWORD=$secret \ + -e MYSQL_ROOT_HOST=% \ + -d mysql:latest \ + >/dev/null + # --require-secure-transport=ON \ + # --ssl-ca /db-ca/certs/ca.crt \ + # --ssl-cert /db-ca/certs/db.crt \ + # --ssl-key /db-ca/private/db.key \ + + local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/ip) + # local host=host.docker.internal + local host=172.17.0.1 + + DB_CONNECTION_TEST=" + docker run + -v $TEST_DIR/db-ca/:/db-ca + --rm mysql + mysql + -h$host + -P$port + -uroot + -p$secret + -e status" \ + timeout --foreground 1m bash -c "wait-for-db-connection" + # --ssl-ca /db-ca/certs/ca.crt + # --ssl-cert /db-ca/certs/client.crt + # --ssl-key /db-ca/private/client.key + + echo "Started $LABEL_SUFFIX db @ $host" + export SERVER_ARGS="${SERVER_ARGS} + --datastore-endpoint=mysql://root:$secret@tcp($host:$port)/testdb + " + # --datastore-cafile /db-ca/certs/ca.crt + # --datastore-certfile /db-ca/certs/client.crt + # --datastore-keyfile /db-ca/private/client.key +} +export -f cluster-pre-hook diff --git a/scripts/test-setup-sonobuoy-postgres b/scripts/test-setup-sonobuoy-postgres new file mode 100644 index 0000000000..89e63131de --- /dev/null +++ b/scripts/test-setup-sonobuoy-postgres @@ -0,0 +1,56 @@ +#!/bin/bash + +. ./scripts/test-setup-sonobuoy + +# --- + +cluster-pre-hook() { + # gen-certs + + mkdir -p $TEST_DIR/db/$LABEL_SUFFIX/metadata + local testID=$(basename $TEST_DIR) + local name=$(echo $LABEL_SUFFIX-$testID | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/name) + local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/port) + local secret=$(echo "${RANDOM}${RANDOM}${RANDOM}" | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/secret) + + docker run --name $name \ + --privileged \ + -p 0.0.0.0:$port:5432 \ + -v $TEST_DIR/db-ca/:/db-ca \ + -e POSTGRES_USER=root \ + -e POSTGRES_PASSWORD=$secret \ + -d postgres:latest \ + >/dev/null + # -c ssl=on \ + # -c ssl_ca_file=/db-ca/certs/ca.crt \ + # -c ssl_cert_file=/db-ca/certs/db.crt \ + # -c ssl_key_file=/db-ca/private/db.key \ + + local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/ip) + # local host=host.docker.internal + local host=172.17.0.1 + + DB_CONNECTION_TEST=" + docker run + -v $TEST_DIR/db-ca/:/db-ca + -e PGPASSWORD=$secret + --rm postgres + psql + -h $host + -p $port + -U root + -c \conninfo" \ + timeout --foreground 1m bash -c "wait-for-db-connection" + # --set sslrootcert=/db-ca/certs/ca.crt + # --set sslcert=/db-ca/certs/client.crt + # --set sslkey=/db-ca/private/client.key + + echo "Started $LABEL_SUFFIX db @ $host" + export SERVER_ARGS="${SERVER_ARGS} + --datastore-endpoint=postgres://root:$secret@$host:$port/testdb?sslmode=disable + " + # --datastore-cafile /db-ca/certs/ca.crt + # --datastore-certfile /db-ca/certs/client.crt + # --datastore-keyfile /db-ca/private/client.key +} +export -f cluster-pre-hook