From 52efc5403b05caebc6b5ecf5db68f3d35b9dcfa3 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Mon, 29 Apr 2019 13:54:51 -0700 Subject: [PATCH] Add e2e testing --- .drone.yml | 19 +++++ .gitignore | 2 +- Dockerfile.sonobuoy.dapper | 25 ++++++ scripts/sonobuoy | 149 +++++++++++++++++++++++++++++++++++ scripts/sonobuoy-config.json | 7 ++ scripts/sonobuoy-e2e-tests | 45 +++++++++++ 6 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.sonobuoy.dapper create mode 100755 scripts/sonobuoy create mode 100644 scripts/sonobuoy-config.json create mode 100755 scripts/sonobuoy-e2e-tests diff --git a/.drone.yml b/.drone.yml index aace97541b..c73f24cec2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,6 +15,25 @@ steps: - name: docker path: /var/run/docker.sock +- name: sonobuoy-e2e-tests + image: rancher/dapper:v0.4.1 + commands: + - dapper -f Dockerfile.sonobuoy.dapper sonobuoy-e2e-tests + volumes: + - name: docker + path: /var/run/docker.sock + when: + instance: + - drone-publish.rancher.io + ref: + include: + - refs/head/master + - refs/tags/* + exclude: + - refs/tags/*-k3s.* + event: + - tag + - name: github_binary_release image: ibuildthecloud/github-release:v0.0.1 settings: diff --git a/.gitignore b/.gitignore index d1f88d56ed..27e91940b5 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ __pycache__ /tests/.pytest_cache/ /tests/.tox/ /tests/.vscode - +/sonobuoy-output diff --git a/Dockerfile.sonobuoy.dapper b/Dockerfile.sonobuoy.dapper new file mode 100644 index 0000000000..058f3753a6 --- /dev/null +++ b/Dockerfile.sonobuoy.dapper @@ -0,0 +1,25 @@ +FROM golang:1.12.1-alpine3.9 + +RUN apk -U --no-cache add bash git docker curl jq coreutils +RUN go get -d github.com/heptio/sonobuoy && \ + git -C /go/src/github.com/heptio/sonobuoy checkout -b current v0.14.2 && \ + go install github.com/heptio/sonobuoy +RUN rm -rf /go/src /go/pkg + +ARG DAPPER_HOST_ARCH +ENV ARCH $DAPPER_HOST_ARCH + +RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \ + curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt \ + )/bin/linux/${ARCH}/kubectl -o /usr/local/bin/kubectl && \ + chmod a+x /usr/local/bin/kubectl + +ENV DAPPER_RUN_ARGS --privileged --network host +ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/ +ENV DAPPER_OUTPUT ./dist +ENV DAPPER_DOCKER_SOCKET true +ENV HOME ${DAPPER_SOURCE} +WORKDIR ${DAPPER_SOURCE} + +ENTRYPOINT ["./scripts/entry.sh"] +CMD ["ci"] diff --git a/scripts/sonobuoy b/scripts/sonobuoy new file mode 100755 index 0000000000..1783c36142 --- /dev/null +++ b/scripts/sonobuoy @@ -0,0 +1,149 @@ +#!/bin/bash + +set -xe + +cd $(dirname $0)/.. + +if [ -z "${K3S_IMAGE}" ]; then + echo "K3S_IMAGE environment variable should be defined" + exit 1 +fi + +# --- + +port-used() { + (cat /dev/tcp/127.0.0.1/$1) 2>/dev/null +} +export -f port-used + +get-port() { + while + PORT=$((10000 + RANDOM % 50000)) + port-used ${PORT} + do continue; done + echo ${PORT} +} +export -f get-port + +K3S_PORT=$(timeout 5s bash -c get-port) +OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT} +mkdir -p ${OUTPUT} + +SECRET=random-$((100000 + RANDOM % 999999)) +export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT} +export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT} +export KUBECONFIG=${OUTPUT}/kubeconfig.yaml + +# --- + +cleanup() { + exit_status=$? + set +e + echo "Cleaning up" + docker logs ${K3S_SERVER} >${OUTPUT}/k3s-server.log 2>&1 + docker logs ${K3S_AGENT} >${OUTPUT}/k3s-agent.log 2>&1 + docker rm -f ${K3S_SERVER} 2>/dev/null + docker rm -f ${K3S_AGENT} 2>/dev/null + rm ${KUBECONFIG} + exit ${exit_status} +} +trap cleanup EXIT + +# --- + +docker run -d --name ${K3S_SERVER} --privileged \ + -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \ + -e K3S_CLUSTER_SECRET=${SECRET} \ + ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT} + +K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER}) +echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}" + +# --- + +docker run -d --name ${K3S_AGENT} --privileged \ + -e K3S_CLUSTER_SECRET=${SECRET} \ + -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \ + ${K3S_IMAGE} agent + +echo "Started ${K3S_AGENT}" + +# --- + +fetch-kubeconfig() { + docker exec ${K3S_SERVER} cat /etc/rancher/k3s/k3s.yaml 2>/dev/null \ + | tee ${KUBECONFIG} +} +export -f fetch-kubeconfig + +wait-for-kubeconfig() { + while [[ -z "$(fetch-kubeconfig)" ]]; do + echo "Waiting for kubeconfig to become available..." + sleep 5 + done +} +export -f wait-for-kubeconfig + +timeout 1m bash -c wait-for-kubeconfig + +# --- + +count-ready-nodes() { + kubectl get nodes -o json \ + | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \ + | wc -l \ + | tr -d '[:space:]' +} +export -f count-ready-nodes + +wait-for-nodes() { + while [[ "$(count-ready-nodes)" != "2" ]]; do + echo "Waiting for nodes to be ready..." + sleep 5 + done +} +export -f wait-for-nodes + +timeout 1m bash -c wait-for-nodes + +# --- + +pod-ready() { + kubectl get pods -n kube-system -o json \ + | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null +} +export -f pod-ready + +wait-for-services() { + for service in coredns; do + while [[ "$(pod-ready ${service})" != "true" ]]; do + echo "Waiting for service ${service} to be ready..." + sleep 5 + done + echo "Service ${service} is ready" + done +} +export -f wait-for-services + +timeout 1m bash -c wait-for-services + +# --- + +echo "Starting sonobuoy tests" + +timeout 30m sonobuoy run \ + --config scripts/sonobuoy-config.json \ + --wait \ + ${@} +sonobuoy status +sonobuoy retrieve ${OUTPUT} + +( + cd ${OUTPUT} + tar xzf *_sonobuoy_*.tar.gz + results="./plugins/e2e/results/e2e.log" + [ -s ${results} ] || exit 1 + if [ -n "${E2E_LOG_OUTPUT}" ]; then + cp ${results} ${E2E_LOG_OUTPUT} + fi +) diff --git a/scripts/sonobuoy-config.json b/scripts/sonobuoy-config.json new file mode 100644 index 0000000000..99885fe5f8 --- /dev/null +++ b/scripts/sonobuoy-config.json @@ -0,0 +1,7 @@ +{ + "Plugins": [ + { + "name": "e2e" + } + ] +} diff --git a/scripts/sonobuoy-e2e-tests b/scripts/sonobuoy-e2e-tests new file mode 100755 index 0000000000..e5603948bb --- /dev/null +++ b/scripts/sonobuoy-e2e-tests @@ -0,0 +1,45 @@ +#!/bin/bash +set -e -x + +source $(dirname $0)/version.sh + +cd $(dirname $0)/.. + +TAG=${TAG:-${VERSION}${SUFFIX}} +REPO=${REPO:-rancher} +IMAGE_NAME=${IMAGE_NAME:-k3s} +export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG} + +OUTPUT=$(pwd)/dist/artifacts +mkdir -p ${OUTPUT} + +pids=() +output=() + +run-sonobuoy() { + output+=(${log_output}) + E2E_LOG_OUTPUT=${log_output} ./scripts/sonobuoy ${@} & + pids+=($!) +} + +log_output=${OUTPUT}/e2e-${ARCH}-serial.log \ + run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]' + +log_output=${OUTPUT}/e2e-${ARCH}-parallel.log \ + run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y + +cleanup() { + exit_status=$? + set +e + echo "Killing the tests!" + kill ${pids[@]} + wait ${pids[@]} + exit ${exit_status} +} +trap cleanup EXIT +wait ${pids[@]} +trap - EXIT + +for log in ${output[@]}; do + tail -20 ${log} +done