mirror of
https://github.com/k3s-io/k3s.git
synced 2024-06-07 19:41:36 +00:00
Test cleanup
This commit is contained in:
Erik Wilson
parent
0867ee1cd0
commit
4439a7df08
12
.drone.yml
12
.drone.yml
@ -59,14 +59,14 @@ steps:
|
||||
event:
|
||||
- tag
|
||||
|
||||
- name: sonobuoy-e2e-tests
|
||||
- name: test
|
||||
image: rancher/dapper:v0.4.2
|
||||
secrets: [ gcloud_auth ]
|
||||
environment:
|
||||
GCLOUD_AUTH:
|
||||
from_secret: gcloud_auth
|
||||
commands:
|
||||
- dapper -f Dockerfile.sonobuoy.dapper
|
||||
- dapper -f Dockerfile.test.dapper
|
||||
volumes:
|
||||
- name: docker
|
||||
path: /var/run/docker.sock
|
||||
@ -154,14 +154,14 @@ steps:
|
||||
event:
|
||||
- tag
|
||||
|
||||
- name: sonobuoy-e2e-tests
|
||||
- name: test
|
||||
image: rancher/dapper:v0.4.2
|
||||
secrets: [ gcloud_auth ]
|
||||
environment:
|
||||
GCLOUD_AUTH:
|
||||
from_secret: gcloud_auth
|
||||
commands:
|
||||
- dapper -f Dockerfile.sonobuoy.dapper
|
||||
- dapper -f Dockerfile.test.dapper
|
||||
volumes:
|
||||
- name: docker
|
||||
path: /var/run/docker.sock
|
||||
@ -232,14 +232,14 @@ steps:
|
||||
event:
|
||||
- tag
|
||||
|
||||
- name: sonobuoy-e2e-tests
|
||||
- name: test
|
||||
image: rancher/dapper:v0.4.2
|
||||
secrets: [ gcloud_auth ]
|
||||
environment:
|
||||
GCLOUD_AUTH:
|
||||
from_secret: gcloud_auth
|
||||
commands:
|
||||
- dapper -f Dockerfile.sonobuoy.dapper
|
||||
- dapper -f Dockerfile.test.dapper
|
||||
volumes:
|
||||
- name: docker
|
||||
path: /var/run/docker.sock
|
||||
|
||||
@ -1,22 +1,23 @@
|
||||
FROM golang:1.13.5-alpine3.10
|
||||
|
||||
RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2
|
||||
RUN SONOBUOY_VER=v0.16.2 && \
|
||||
SONOBUOY_PKG=github.com/vmware-tanzu/sonobuoy && \
|
||||
go get -d ${SONOBUOY_PKG} && \
|
||||
cd /go/src/${SONOBUOY_PKG} && \
|
||||
git checkout -b current ${SONOBUOY_VER} && \
|
||||
go build -o /usr/local/bin/sonobuoy
|
||||
RUN rm -rf /go/src /go/pkg
|
||||
RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2 openssl
|
||||
|
||||
ARG DAPPER_HOST_ARCH
|
||||
ENV ARCH $DAPPER_HOST_ARCH
|
||||
|
||||
RUN if [ "${ARCH}" == "amd64" ] || [ "${ARCH}" == "arm64" ]; then \
|
||||
VERSION=0.17.0 OS=linux && \
|
||||
curl -sL "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_${ARCH}.tar.gz" | \
|
||||
tar -xzf - -C /usr/local/bin; \
|
||||
fi
|
||||
|
||||
RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \
|
||||
curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt \
|
||||
)/bin/linux/${ARCH}/kubectl -o /usr/local/bin/kubectl && \
|
||||
chmod a+x /usr/local/bin/kubectl
|
||||
|
||||
ENV TEST_CLEANUP true
|
||||
|
||||
ENV DAPPER_RUN_ARGS --privileged --network host
|
||||
ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME GCLOUD_AUTH
|
||||
ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
|
||||
@ -26,4 +27,4 @@ ENV HOME ${DAPPER_SOURCE}
|
||||
WORKDIR ${DAPPER_SOURCE}
|
||||
|
||||
ENTRYPOINT ["./scripts/entry.sh"]
|
||||
CMD ["sonobuoy-e2e-tests"]
|
||||
CMD ["test"]
|
||||
@ -40,7 +40,7 @@ END
|
||||
HOME=${TMPDIR}
|
||||
PATH=$PATH:${HOME}/gsutil
|
||||
|
||||
LOG_TGZ=k3s-log-$(uname -s | tr '[:upper:]' '[:lower:]' )-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(date +%s)-$(basename $1).tgz
|
||||
LOG_TGZ=k3s-log-$(date +%s)-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(basename $1).tgz
|
||||
|
||||
tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1)
|
||||
gsutil cp ${TMPDIR}/${LOG_TGZ} gs://k3s-ci-logs || exit 1
|
||||
|
||||
@ -1,82 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -xe
|
||||
|
||||
cd $(dirname $0)/..
|
||||
|
||||
. ./scripts/test-helpers
|
||||
|
||||
if [ -z "${K3S_IMAGE}" ]; then
|
||||
echo 'K3S_IMAGE environment variable should be defined'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---
|
||||
|
||||
cleanup() {
|
||||
exit_status=$?
|
||||
set +e
|
||||
echo 'Cleaning up'
|
||||
trap - EXIT
|
||||
[ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null
|
||||
if [ "${exit_status}" -ne "0" ]; then
|
||||
dump-logs
|
||||
fi
|
||||
docker rm -f ${CONTAINERS} 2>/dev/null
|
||||
rm ${KUBECONFIG}
|
||||
exit ${exit_status}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ---
|
||||
|
||||
K3S_PORT=$(timeout --foreground 5s bash -c get-port)
|
||||
OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
|
||||
LOGS=$(pwd)/logs/$$
|
||||
E2E="${OUTPUT}/e2e"
|
||||
E2E_LOG='plugins/e2e/results/global/e2e.log'
|
||||
RESULTS="${E2E}/${E2E_LOG}"
|
||||
|
||||
SECRET=random-$((100000 + RANDOM % 999999))
|
||||
export K3S_AGENT=k3s-agent-${K3S_PORT}
|
||||
export K3S_SERVER=k3s-server-${K3S_PORT}
|
||||
export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
|
||||
export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
|
||||
|
||||
mkdir -p ${OUTPUT}
|
||||
|
||||
# ---
|
||||
|
||||
docker run -d --name ${K3S_SERVER} --privileged \
|
||||
-p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
|
||||
-e K3S_CLUSTER_SECRET=${SECRET} \
|
||||
-e K3S_DEBUG=true \
|
||||
${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}
|
||||
|
||||
K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
|
||||
echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}"
|
||||
|
||||
docker exec ${K3S_SERVER} check-config || true
|
||||
timeout --foreground 1m bash -c wait-for-kubeconfig
|
||||
verify-valid-versions ${K3S_SERVER}
|
||||
|
||||
# ---
|
||||
|
||||
docker run -d --name ${K3S_AGENT} --privileged \
|
||||
-e K3S_CLUSTER_SECRET=${SECRET} \
|
||||
-e K3S_URL=https://${K3S_IP}:${K3S_PORT} \
|
||||
${K3S_IMAGE} agent
|
||||
|
||||
echo "Started ${K3S_AGENT}"
|
||||
|
||||
# ---
|
||||
|
||||
timeout --foreground 1m bash -c 'wait-for-nodes 2'
|
||||
timeout --foreground 3m bash -c 'wait-for-services coredns local-path-provisioner metrics-server'
|
||||
|
||||
if [ "$ARCH" = 'arm' ]; then
|
||||
echo "Aborting sonobuoy tests, images not available for $ARCH"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo 'Starting sonobuoy tests'
|
||||
sonobuoy-test "${@}"
|
||||
@ -1,83 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e -x
|
||||
|
||||
cd $(dirname $0)/..
|
||||
|
||||
if [ -z "$K3S_IMAGE" ]; then
|
||||
. ./scripts/version.sh
|
||||
TAG=${TAG:-${VERSION_TAG}${SUFFIX}}
|
||||
REPO=${REPO:-rancher}
|
||||
IMAGE_NAME=${IMAGE_NAME:-k3s}
|
||||
export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG}
|
||||
fi
|
||||
|
||||
OUTPUT=$(pwd)/dist/artifacts
|
||||
mkdir -p ${OUTPUT}
|
||||
|
||||
pids=()
|
||||
output=()
|
||||
|
||||
show-logs() {
|
||||
for pid in "${pids[@]}"; do
|
||||
logdir=$(pwd)/logs/${pid}
|
||||
if [ ! -d $logdir ]; then
|
||||
continue
|
||||
fi
|
||||
echo
|
||||
echo "#- Begin: logs for sonobuoy run pid ${pid}"
|
||||
for log in $(pwd)/logs/${pid}/*.log; do
|
||||
if [ -f ${log} ]; then
|
||||
echo
|
||||
echo "#- Tail: ${log}"
|
||||
tail -10 ${log}
|
||||
echo "#- Done: ${log}"
|
||||
echo
|
||||
fi
|
||||
done
|
||||
echo "#- Finish: logs for sonobuoy run pid ${pid}"
|
||||
echo
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
exit_status=$?
|
||||
set +e
|
||||
trap - EXIT INT
|
||||
kill ${pids[@]} 2>/dev/null
|
||||
wait
|
||||
set +x
|
||||
echo -n "Tests "
|
||||
if [ "${exit_status}" -eq "0" ]; then
|
||||
echo "passed"
|
||||
else
|
||||
echo "failed"
|
||||
show-logs
|
||||
fi
|
||||
exit ${exit_status}
|
||||
}
|
||||
trap cleanup EXIT INT
|
||||
|
||||
|
||||
# ---
|
||||
|
||||
run-sonobuoy() {
|
||||
output+=(${log_output})
|
||||
E2E_LOG_OUTPUT=${log_output} \
|
||||
./scripts/sonobuoy ${@} \
|
||||
> >(stdbuf -oL sed "s/^/[${label}] /") 2>&1 &
|
||||
pids+=($!)
|
||||
}
|
||||
|
||||
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log label=PARALLEL \
|
||||
run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y
|
||||
|
||||
sleep 60
|
||||
|
||||
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log label=SERIAL \
|
||||
run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
|
||||
|
||||
exit_code=0
|
||||
for pid in "${pids[@]}"; do
|
||||
wait $pid || exit_code=$?
|
||||
done
|
||||
exit ${exit_code}
|
||||
29
scripts/test
Executable file
29
scripts/test
Executable file
@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
set -e -x
|
||||
cd $(dirname $0)/..
|
||||
|
||||
. ./scripts/test-helpers
|
||||
|
||||
artifacts=$(pwd)/dist/artifacts
|
||||
mkdir -p $artifacts
|
||||
|
||||
# ---
|
||||
|
||||
. ./scripts/test-run-basics
|
||||
|
||||
# # ---
|
||||
|
||||
E2E_OUTPUT=$artifacts test-run-sonobuoy
|
||||
|
||||
# # ---
|
||||
|
||||
# if [ "$ARCH" != 'amd64' ]; then
|
||||
# printf "\033[33mSkipping remaining tests, images not available for $ARCH.\033[m\n"
|
||||
# exit 0
|
||||
# fi
|
||||
|
||||
# # ---
|
||||
|
||||
# test-run-sonobuoy mysql
|
||||
# test-run-sonobuoy postgres
|
||||
# test-run-sonobuoy dqlite
|
||||
72
scripts/test-certs-openssl.cnf
Normal file
72
scripts/test-certs-openssl.cnf
Normal file
@ -0,0 +1,72 @@
|
||||
# db OpenSSL configuration file.
|
||||
SAN = "IP:127.0.0.1, IP:172.17.0.1, DNS:host.docker.internal"
|
||||
dir = .
|
||||
|
||||
[ ca ]
|
||||
default_ca = db_ca
|
||||
|
||||
[ db_ca ]
|
||||
certs = $dir/certs
|
||||
certificate = $dir/certs/ca.crt
|
||||
crl = $dir/crl.pem
|
||||
crl_dir = $dir/crl
|
||||
crlnumber = $dir/crlnumber
|
||||
database = $dir/index.txt
|
||||
email_in_dn = no
|
||||
new_certs_dir = $dir/newcerts
|
||||
private_key = $dir/private/ca.key
|
||||
serial = $dir/serial
|
||||
RANDFILE = $dir/private/.rand
|
||||
name_opt = ca_default
|
||||
cert_opt = ca_default
|
||||
default_days = 3650
|
||||
default_crl_days = 30
|
||||
default_md = sha512
|
||||
preserve = no
|
||||
policy = policy_db
|
||||
|
||||
[ policy_db ]
|
||||
organizationName = optional
|
||||
commonName = supplied
|
||||
|
||||
[ req ]
|
||||
default_bits = 1024
|
||||
default_keyfile = privkey.pem
|
||||
distinguished_name = req_distinguished_name
|
||||
attributes = req_attributes
|
||||
x509_extensions = v3_ca
|
||||
string_mask = utf8only
|
||||
req_extensions = db_client
|
||||
|
||||
[ req_distinguished_name ]
|
||||
countryName = Country Name (2 letter code)
|
||||
countryName_default = US
|
||||
countryName_min = 2
|
||||
countryName_max = 2
|
||||
commonName = Common Name (FQDN)
|
||||
0.organizationName = Organization Name (eg, company)
|
||||
0.organizationName_default = db-ca
|
||||
|
||||
[ req_attributes ]
|
||||
|
||||
[ v3_ca ]
|
||||
basicConstraints = CA:true
|
||||
keyUsage = keyCertSign,cRLSign
|
||||
subjectKeyIdentifier = hash
|
||||
|
||||
[ db_client ]
|
||||
basicConstraints = CA:FALSE
|
||||
extendedKeyUsage = clientAuth
|
||||
keyUsage = digitalSignature, keyEncipherment
|
||||
|
||||
[ db_peer ]
|
||||
basicConstraints = CA:FALSE
|
||||
extendedKeyUsage = clientAuth, serverAuth
|
||||
keyUsage = digitalSignature, keyEncipherment
|
||||
subjectAltName = ${ENV::SAN}
|
||||
|
||||
[ db_server ]
|
||||
basicConstraints = CA:FALSE
|
||||
extendedKeyUsage = clientAuth, serverAuth
|
||||
keyUsage = digitalSignature, keyEncipherment
|
||||
subjectAltName = ${ENV::SAN}
|
||||
@ -10,26 +10,31 @@ export -f port-used
|
||||
# ---
|
||||
|
||||
get-port() {
|
||||
local port=0
|
||||
while
|
||||
PORT=$((10000 + RANDOM % 50000))
|
||||
port-used ${PORT}
|
||||
port=$((10000 + RANDOM % 50000))
|
||||
port-used $port
|
||||
do continue; done
|
||||
echo ${PORT}
|
||||
echo $port
|
||||
}
|
||||
export -f get-port
|
||||
|
||||
# ---
|
||||
|
||||
fetch-kubeconfig() {
|
||||
docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
|
||||
}
|
||||
fetch-kubeconfig() {(
|
||||
set -e -o pipefail
|
||||
local num=${1:-1}
|
||||
local name=$(cat $TEST_DIR/servers/$num/metadata/name)
|
||||
local port=$(cat $TEST_DIR/servers/$num/metadata/port)
|
||||
docker cp $name:/etc/rancher/k3s/k3s.yaml - 2>/dev/null | tar -xO 2>/dev/null | sed -e "s/:6443/:$port/g" >$TEST_DIR/servers/$num/kubeconfig.yaml
|
||||
)}
|
||||
export -f fetch-kubeconfig
|
||||
|
||||
# ---
|
||||
|
||||
wait-for-kubeconfig() {
|
||||
while ! fetch-kubeconfig; do
|
||||
echo 'Waiting for kubeconfig to become available...'
|
||||
while ! fetch-kubeconfig $1; do
|
||||
echo 'Waiting for kubeconfig to become available...' >&2
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
@ -49,7 +54,7 @@ export -f count-ready-nodes
|
||||
|
||||
wait-for-nodes() {
|
||||
while [[ $(count-ready-nodes) -ne $1 ]]; do
|
||||
echo 'Waiting for nodes to be ready...'
|
||||
echo 'Waiting for nodes to be ready...' >&2
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
@ -67,17 +72,31 @@ export -f pod-ready
|
||||
|
||||
wait-for-services() {
|
||||
for service in $@; do
|
||||
while [[ "$(pod-ready ${service})" != 'true' ]]; do
|
||||
echo "Waiting for service ${service} to be ready..."
|
||||
while [[ "$(pod-ready $service)" != 'true' ]]; do
|
||||
echo "Waiting for service $service to be ready..." >&2
|
||||
sleep 5
|
||||
done
|
||||
echo "Service ${service} is ready"
|
||||
echo "Service $service is ready"
|
||||
done
|
||||
}
|
||||
export -f wait-for-services
|
||||
|
||||
# ---
|
||||
|
||||
wait-for-db-connection() {
|
||||
if [ -z "$DB_CONNECTION_TEST" ]; then
|
||||
echo 'DB_CONNECTION_TEST is not defined' >&2
|
||||
return 1
|
||||
fi
|
||||
while ! $DB_CONNECTION_TEST 2>/dev/null; do
|
||||
echo 'Waiting for database to become available...' >&2
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
export -f wait-for-db-connection
|
||||
|
||||
# ---
|
||||
|
||||
verify-valid-version() {
|
||||
if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
|
||||
return 1
|
||||
@ -97,13 +116,30 @@ export -f verify-valid-versions
|
||||
# ---
|
||||
|
||||
dump-logs() {
|
||||
mkdir -p ${LOGS}/sonobuoy
|
||||
tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${LOGS}/sonobuoy
|
||||
for container in ${CONTAINERS}; do
|
||||
docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/${container}-containerd.log
|
||||
docker logs ${container} >${LOGS}/${container}-system.log 2>&1
|
||||
local testID=$(basename $TEST_DIR)
|
||||
echo "#---------------------------------"
|
||||
echo "#- Begin: logs for run ($testID)"
|
||||
echo
|
||||
for node in $TEST_DIR/*/*; do
|
||||
[ -d "$node" ] || continue
|
||||
local name=$(cat $node/metadata/name 2>/dev/null)
|
||||
[ "$name" ] || continue
|
||||
mkdir -p $node/logs
|
||||
docker cp $name:/var/lib/rancher/k3s/agent/containerd/containerd.log $node/logs/containerd.log 2>/dev/null
|
||||
docker logs $name >$node/logs/system.log 2>&1
|
||||
for log in $node/logs/*.log; do
|
||||
echo
|
||||
echo "#- Tail: $log"
|
||||
tail -5 $log
|
||||
echo "#- Done: $log"
|
||||
echo
|
||||
done
|
||||
done
|
||||
./scripts/log-upload ${LOGS}
|
||||
echo
|
||||
echo "#- Finish: logs for run ($testID)"
|
||||
echo "#---------------------------------"
|
||||
echo
|
||||
./scripts/log-upload $TEST_DIR
|
||||
}
|
||||
export -f dump-logs
|
||||
|
||||
@ -111,40 +147,389 @@ export -f dump-logs
|
||||
|
||||
retrieve-sonobuoy-logs() {
|
||||
sonobuoy status || true
|
||||
if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
|
||||
status=passed
|
||||
exit_code=0
|
||||
else
|
||||
|
||||
local status=passed
|
||||
local code=0
|
||||
|
||||
if ! sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
|
||||
status=failed
|
||||
exit_code=1
|
||||
code=1
|
||||
fi
|
||||
|
||||
mkdir -p ${E2E}
|
||||
sonobuoy retrieve ${E2E}
|
||||
tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
|
||||
if [ ! -s ${RESULTS} ]; then
|
||||
mkdir -p $TEST_DIR/sonobuoy
|
||||
sonobuoy retrieve $TEST_DIR/sonobuoy 2>/dev/null || true
|
||||
local logTarball=$TEST_DIR/sonobuoy/*_sonobuoy_*.tar.gz
|
||||
|
||||
if [ -f $logTarball ]; then
|
||||
tar -xz -f $logTarball -C $TEST_DIR/sonobuoy
|
||||
rm $logTarball
|
||||
else
|
||||
rm -rf $TEST_DIR/sonobuoy
|
||||
fi
|
||||
|
||||
local e2eLog=$TEST_DIR/sonobuoy/plugins/e2e/results/global/e2e.log
|
||||
if [ ! -s $e2eLog ]; then
|
||||
return 1
|
||||
fi
|
||||
if [ -n "${E2E_LOG_OUTPUT}" ]; then
|
||||
cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
|
||||
if [ -n "$LOG_OUTPUT" ]; then
|
||||
cp $e2eLog $(sed -e "s/-STATUS-/-$status-/g" <<< "$LOG_OUTPUT")
|
||||
fi
|
||||
awk '/^Summarizing .* Failures?:$/,0' ${RESULTS}
|
||||
return ${exit_code}
|
||||
awk '/^Summarizing .* Failures?:$/,0' $e2eLog
|
||||
return $code
|
||||
}
|
||||
export -f retrieve-sonobuoy-logs
|
||||
|
||||
# ---
|
||||
|
||||
sonobuoy-test() {
|
||||
time sonobuoy run \
|
||||
--config=scripts/sonobuoy-config.json \
|
||||
--plugin-env=e2e.E2E_USE_GO_RUNNER=true \
|
||||
--wait=30 \
|
||||
"${@}" &
|
||||
SONOBUOY_PID=$!
|
||||
wait $SONOBUOY_PID
|
||||
if [ "$ARCH" = 'arm' ]; then
|
||||
echo "Aborting sonobuoy tests, images not available for $ARCH"
|
||||
return 0
|
||||
fi
|
||||
echo 'Starting sonobuoy tests'
|
||||
|
||||
sonobuoy run --config=scripts/sonobuoy-config.json --plugin-env=e2e.E2E_USE_GO_RUNNER=true --wait=30 $@ &
|
||||
local sonobuoyPID=$!
|
||||
local code=0
|
||||
time wait $sonobuoyPID || code=$?
|
||||
|
||||
retrieve-sonobuoy-logs
|
||||
return $code
|
||||
}
|
||||
export -f sonobuoy-test
|
||||
|
||||
# ---
|
||||
|
||||
test-cleanup() {
|
||||
local code=$?
|
||||
set +e +x
|
||||
echo 'Cleaning up...'
|
||||
trap - EXIT INT TERM
|
||||
if [[ $code -ne 0 ]]; then
|
||||
dump-logs
|
||||
fi
|
||||
for name in $TEST_DIR/*/*/metadata/name; do
|
||||
[ -f "$name" ] || continue
|
||||
local container=$(cat $name)
|
||||
docker rm -f -v $container >/dev/null 2>&1 &
|
||||
done
|
||||
if [ "$TEST_CLEANUP" = true ]; then
|
||||
echo "Removing $TEST_DIR"
|
||||
rm -rf $TEST_DIR >/dev/null 2>&1 &
|
||||
fi
|
||||
[ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK >/dev/null 2>&1 &
|
||||
echo
|
||||
echo -n "Test $(basename $TEST_DIR) "
|
||||
if [ $code -eq 0 ]; then
|
||||
echo "passed."
|
||||
else
|
||||
echo "failed."
|
||||
fi
|
||||
echo
|
||||
exit $code
|
||||
}
|
||||
export -f test-cleanup
|
||||
|
||||
# ---
|
||||
|
||||
test-setup() {
|
||||
export TEST_DIR=$(mktemp -d '/tmp/XXXXXX')
|
||||
trap test-cleanup EXIT INT TERM
|
||||
|
||||
mkdir -p $TEST_DIR/metadata
|
||||
if [ "$LABEL" ]; then
|
||||
exec > >(awk "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \
|
||||
2> >(awk "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2)
|
||||
echo "$LABEL" >$TEST_DIR/metadata/label
|
||||
fi
|
||||
|
||||
mkdir -p $TEST_DIR/logs
|
||||
exec > >(tee -a $TEST_DIR/logs/test.log) \
|
||||
2> >(tee -a $TEST_DIR/logs/test.log >&2)
|
||||
|
||||
if [ -z "$K3S_IMAGE" ]; then
|
||||
. ./scripts/version.sh
|
||||
TAG=${TAG:-${VERSION_TAG}${SUFFIX}}
|
||||
REPO=${REPO:-rancher}
|
||||
IMAGE_NAME=${IMAGE_NAME:-k3s}
|
||||
export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG}
|
||||
fi
|
||||
|
||||
if [ -z "$K3S_IMAGE" ]; then
|
||||
echo 'K3S_IMAGE environment variable should be defined'
|
||||
return 1
|
||||
fi
|
||||
|
||||
local setupFile=./scripts/test-setup-${TEST_TYPE}
|
||||
[ -f $setupFile ] && source $setupFile
|
||||
|
||||
echo ${RANDOM}${RANDOM}${RANDOM} >$TEST_DIR/metadata/secret
|
||||
}
|
||||
export -f test-setup
|
||||
|
||||
# ---
|
||||
|
||||
gen-certs() {(
|
||||
set -e -x
|
||||
#umask 077
|
||||
|
||||
local opensslConfig=$(pwd)/scripts/test-certs-openssl.cnf
|
||||
local subject="/C=US/ST=AZ/L=Tempe/O=Rancher/OU=DevOps/DC=com/DC=rancher"
|
||||
local caDir=$TEST_DIR/db-ca
|
||||
[ -d $caDir ] && rm -rf $caDir
|
||||
mkdir -p $caDir
|
||||
|
||||
cd $caDir
|
||||
mkdir -p private certs newcerts crl
|
||||
touch index.txt
|
||||
echo '01' > serial
|
||||
|
||||
openssl req \
|
||||
-config $opensslConfig \
|
||||
-new -x509 \
|
||||
-nodes \
|
||||
-subj $subject \
|
||||
-extensions v3_ca \
|
||||
-keyout private/ca.key \
|
||||
-out certs/ca.crt
|
||||
|
||||
openssl req \
|
||||
-config $opensslConfig \
|
||||
-new \
|
||||
-nodes \
|
||||
-subj $subject"/CN=server" \
|
||||
-keyout private/db.key \
|
||||
-out db.csr
|
||||
|
||||
openssl ca \
|
||||
-config $opensslConfig \
|
||||
-batch \
|
||||
-extensions db_server \
|
||||
-keyfile private/ca.key \
|
||||
-cert certs/ca.crt \
|
||||
-out certs/db.crt \
|
||||
-infiles db.csr
|
||||
|
||||
openssl req \
|
||||
-config $opensslConfig \
|
||||
-new \
|
||||
-nodes \
|
||||
-subj $subject"/CN=client" \
|
||||
-keyout private/client.key \
|
||||
-out client.csr
|
||||
|
||||
openssl ca \
|
||||
-config $opensslConfig \
|
||||
-batch \
|
||||
-extensions db_client \
|
||||
-keyfile private/ca.key \
|
||||
-cert certs/ca.crt \
|
||||
-out certs/client.crt \
|
||||
-infiles client.csr
|
||||
)}
|
||||
export -f gen-certs
|
||||
|
||||
# ---
|
||||
|
||||
inc-count() {(
|
||||
shopt -s extglob
|
||||
local count=$(exec 2>/dev/null; ls -1d $TEST_DIR/$1/+([0-9]) | xargs -n1 basename | sort -n -r | head -1)
|
||||
count=$((count+1))
|
||||
mkdir -p $TEST_DIR/$1/$count/metadata
|
||||
echo $count
|
||||
)}
|
||||
export -f inc-count
|
||||
|
||||
# ---
|
||||
|
||||
run-function() {
|
||||
declare -f $1 >/dev/null 2>&1 || return 0
|
||||
$@
|
||||
}
|
||||
export -f run-function
|
||||
|
||||
# ---
|
||||
|
||||
provision-server() {
|
||||
local count=$(inc-count servers)
|
||||
local testID=$(basename $TEST_DIR)
|
||||
local name=$(echo "k3s-server-$count-$testID" | tee $TEST_DIR/servers/$count/metadata/name)
|
||||
#local args=$(cat $TEST_DIR/args $TEST_DIR/servers/args $TEST_DIR/servers/$count/args 2>/dev/null)
|
||||
local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/servers/$count/metadata/port)
|
||||
local SERVER_INSTANCE_ARGS="SERVER_${count}_ARGS"
|
||||
|
||||
run-function server-pre-hook $count
|
||||
|
||||
docker run \
|
||||
-d --name $name \
|
||||
--privileged \
|
||||
-v $TEST_DIR/db-ca/:/db-ca \
|
||||
-p 127.0.0.1:$port:6443 \
|
||||
-p 6443 \
|
||||
-e K3S_TOKEN=$(cat $TEST_DIR/metadata/secret) \
|
||||
-e K3S_DEBUG=true \
|
||||
$K3S_IMAGE server $ARGS $SERVER_ARGS ${!SERVER_INSTANCE_ARGS}
|
||||
|
||||
local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/servers/$count/metadata/ip)
|
||||
local url=$(echo "https://$ip:6443" | tee $TEST_DIR/servers/$count/metadata/url)
|
||||
|
||||
echo "Started $name @ $url"
|
||||
run-function server-post-hook $count
|
||||
}
|
||||
export -f provision-server
|
||||
|
||||
# ---
|
||||
|
||||
provision-agent() {
|
||||
local K3S_URL=${K3S_URL:-"$(cat $TEST_DIR/servers/1/metadata/url)"}
|
||||
local count=$(inc-count agents)
|
||||
local testID=$(basename $TEST_DIR)
|
||||
local name=$(echo "k3s-agent-$count-$testID" | tee $TEST_DIR/agents/$count/metadata/name)
|
||||
#local args=$(cat $TEST_DIR/args $TEST_DIR/agents/args $TEST_DIR/agents/$count/args 2>/dev/null)
|
||||
local AGENT_INSTANCE_ARGS="AGENT_${count}_ARGS"
|
||||
|
||||
run-function agent-pre-hook $count
|
||||
docker run \
|
||||
-d --name $name \
|
||||
--privileged \
|
||||
-e K3S_TOKEN=$(cat $TEST_DIR/metadata/secret) \
|
||||
-e K3S_URL=$K3S_URL \
|
||||
$K3S_IMAGE agent $ARGS $AGENT_ARGS ${!AGENT_INSTANCE_ARGS}
|
||||
|
||||
echo "Started $name"
|
||||
run-function agent-post-hook $count
|
||||
}
|
||||
export -f provision-agent
|
||||
|
||||
# ---
|
||||
|
||||
provision-cluster() {
|
||||
run-function cluster-pre-hook
|
||||
|
||||
for i in $(seq 1 $NUM_SERVERS); do
|
||||
provision-server
|
||||
timeout --foreground 30s bash -c "wait-for-kubeconfig $i"
|
||||
done
|
||||
export KUBECONFIG=$TEST_DIR/servers/1/kubeconfig.yaml
|
||||
|
||||
if [ $NUM_AGENTS -gt 0 ]; then
|
||||
for _ in $(seq 1 $NUM_AGENTS); do
|
||||
provision-agent
|
||||
done
|
||||
fi
|
||||
|
||||
timeout --foreground 1m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))"
|
||||
timeout --foreground 3m bash -c "wait-for-services $WAIT_SERVICES"
|
||||
|
||||
run-function cluster-post-hook
|
||||
|
||||
[ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK
|
||||
}
|
||||
export -f provision-cluster
|
||||
|
||||
# ---
|
||||
|
||||
run-test() {
|
||||
export PROVISION_LOCK=$(mktemp)
|
||||
./scripts/test-runner $@ &
|
||||
pids+=($!)
|
||||
(
|
||||
set +x
|
||||
while [ -f "$PROVISION_LOCK" ]; do
|
||||
sleep 1
|
||||
done
|
||||
sleep 5
|
||||
)
|
||||
}
|
||||
export -f run-test
|
||||
|
||||
# ---
|
||||
|
||||
e2e-test() {
|
||||
local label=$label
|
||||
if [ -n "$LABEL_SUFFIX" ]; then
|
||||
label="$label-$LABEL_SUFFIX"
|
||||
fi
|
||||
local logOutput=
|
||||
if [ -n "$E2E_OUTPUT" ]; then
|
||||
logOutput=$E2E_OUTPUT/$logname
|
||||
fi
|
||||
LABEL=$label LOG_OUTPUT=$logOutput run-test $@
|
||||
}
|
||||
|
||||
# ---
|
||||
|
||||
run-e2e-tests() {
|
||||
label=PARALLEL \
|
||||
logName=e2e-STATUS-${ARCH}-parallel.log \
|
||||
e2e-test ${sonobuoyParallelArgs[@]}
|
||||
|
||||
label=SERIAL \
|
||||
logName=e2e-STATUS-${ARCH}-serial.log \
|
||||
e2e-test ${sonobuoySerialArgs[@]}
|
||||
}
|
||||
export -f run-e2e-tests
|
||||
|
||||
# ---
|
||||
|
||||
test-run-sonobuoy() {
|
||||
local suffix
|
||||
if [ "$1" ]; then
|
||||
suffix="-$1"
|
||||
export LABEL_SUFFIX=$1
|
||||
fi
|
||||
|
||||
. ./scripts/test-setup-sonobuoy$suffix
|
||||
run-e2e-tests
|
||||
}
|
||||
export -f test-run-sonobuoy
|
||||
|
||||
# ---
|
||||
|
||||
pid-cleanup() {
|
||||
local code=$?
|
||||
set +e
|
||||
local failCount=0
|
||||
if [ $code -ne 0 ]; then
|
||||
for pid in ${pids[@]}; do
|
||||
pkill -P $pid
|
||||
wait $pid || failCount=$((failCount+1))
|
||||
done
|
||||
fi
|
||||
wait
|
||||
trap - EXIT INT TERM
|
||||
set +x
|
||||
echo
|
||||
if [ $failCount -eq 0 ]; then
|
||||
printf '\033[32mAll tests passed.\033[m\n'
|
||||
if [ $code -ne 0 ]; then
|
||||
printf "\033[31mExit code is $code.\033[m\n"
|
||||
fi
|
||||
else
|
||||
printf "\033[31m$failCount tests failed.\033[m\n"
|
||||
fi
|
||||
echo
|
||||
exit $code
|
||||
}
|
||||
export -f pid-cleanup
|
||||
|
||||
# ---
|
||||
|
||||
wait-pids() {
|
||||
trap - EXIT
|
||||
set +e
|
||||
local code=0
|
||||
for pid in "${pids[@]}"; do
|
||||
wait $pid || code=$?
|
||||
done
|
||||
[ $code -eq 0 ]
|
||||
pid-cleanup
|
||||
exit $code
|
||||
}
|
||||
export -f wait-pids
|
||||
|
||||
# ---
|
||||
|
||||
pids=()
|
||||
trap pid-cleanup INT TERM
|
||||
trap wait-pids EXIT
|
||||
|
||||
14
scripts/test-run-basics
Executable file
14
scripts/test-run-basics
Executable file
@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
|
||||
export NUM_SERVERS=1
|
||||
export NUM_AGENTS=1
|
||||
export SERVER_ARGS='--no-deploy=traefik,coredns,local-storage,metrics-server'
|
||||
|
||||
start-test() {
|
||||
docker exec $(cat $TEST_DIR/servers/1/metadata/name) check-config || true
|
||||
verify-valid-versions $(cat $TEST_DIR/servers/1/metadata/name)
|
||||
}
|
||||
export -f start-test
|
||||
|
||||
# --- create a basic cluster and check for valid versions
|
||||
LABEL=BASICS run-test
|
||||
13
scripts/test-runner
Executable file
13
scripts/test-runner
Executable file
@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
set -x -e
|
||||
cd $(dirname $0)/..
|
||||
|
||||
# ---
|
||||
|
||||
for include in $TEST_INCLUDES; do
|
||||
. $include
|
||||
done
|
||||
|
||||
test-setup
|
||||
provision-cluster
|
||||
start-test $@
|
||||
14
scripts/test-setup-sonobuoy
Executable file
14
scripts/test-setup-sonobuoy
Executable file
@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
|
||||
export NUM_SERVERS=1
|
||||
export NUM_AGENTS=1
|
||||
export SERVER_ARGS='--no-deploy=traefik'
|
||||
export WAIT_SERVICES='coredns local-path-provisioner metrics-server'
|
||||
|
||||
export sonobuoyParallelArgs=(--e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y)
|
||||
export sonobuoySerialArgs=(--e2e-focus='\[Serial\].*\[Conformance\]')
|
||||
|
||||
start-test() {
|
||||
sonobuoy-test $@
|
||||
}
|
||||
export -f start-test
|
||||
18
scripts/test-setup-sonobuoy-dqlite
Normal file
18
scripts/test-setup-sonobuoy-dqlite
Normal file
@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
|
||||
. ./scripts/test-setup-sonobuoy
|
||||
|
||||
export NUM_SERVERS=2
|
||||
export NUM_AGENTS=0
|
||||
|
||||
export SERVER_1_ARGS=--cluster-init
|
||||
|
||||
# ---
|
||||
|
||||
server-post-hook() {
|
||||
if [ $1 -eq 1 ]; then
|
||||
local url=$(cat $TEST_DIR/servers/1/metadata/url)
|
||||
export SERVER_ARGS="--server $url"
|
||||
fi
|
||||
}
|
||||
export -f server-post-hook
|
||||
56
scripts/test-setup-sonobuoy-mysql
Normal file
56
scripts/test-setup-sonobuoy-mysql
Normal file
@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
|
||||
. ./scripts/test-setup-sonobuoy
|
||||
|
||||
# ---
|
||||
|
||||
cluster-pre-hook() {
|
||||
# gen-certs
|
||||
|
||||
mkdir -p $TEST_DIR/db/$LABEL_SUFFIX/metadata
|
||||
local testID=$(basename $TEST_DIR)
|
||||
local name=$(echo $LABEL_SUFFIX-$testID | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/name)
|
||||
local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/port)
|
||||
local secret=$(echo "${RANDOM}${RANDOM}${RANDOM}" | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/secret)
|
||||
|
||||
docker run --name $name \
|
||||
--privileged \
|
||||
-p 0.0.0.0:$port:3306 \
|
||||
-v $TEST_DIR/db-ca/:/db-ca \
|
||||
-e MYSQL_ROOT_PASSWORD=$secret \
|
||||
-e MYSQL_ROOT_HOST=% \
|
||||
-d mysql:latest \
|
||||
>/dev/null
|
||||
# --require-secure-transport=ON \
|
||||
# --ssl-ca /db-ca/certs/ca.crt \
|
||||
# --ssl-cert /db-ca/certs/db.crt \
|
||||
# --ssl-key /db-ca/private/db.key \
|
||||
|
||||
local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/ip)
|
||||
# local host=host.docker.internal
|
||||
local host=172.17.0.1
|
||||
|
||||
DB_CONNECTION_TEST="
|
||||
docker run
|
||||
-v $TEST_DIR/db-ca/:/db-ca
|
||||
--rm mysql
|
||||
mysql
|
||||
-h$host
|
||||
-P$port
|
||||
-uroot
|
||||
-p$secret
|
||||
-e status" \
|
||||
timeout --foreground 1m bash -c "wait-for-db-connection"
|
||||
# --ssl-ca /db-ca/certs/ca.crt
|
||||
# --ssl-cert /db-ca/certs/client.crt
|
||||
# --ssl-key /db-ca/private/client.key
|
||||
|
||||
echo "Started $LABEL_SUFFIX db @ $host"
|
||||
export SERVER_ARGS="${SERVER_ARGS}
|
||||
--datastore-endpoint=mysql://root:$secret@tcp($host:$port)/testdb
|
||||
"
|
||||
# --datastore-cafile /db-ca/certs/ca.crt
|
||||
# --datastore-certfile /db-ca/certs/client.crt
|
||||
# --datastore-keyfile /db-ca/private/client.key
|
||||
}
|
||||
export -f cluster-pre-hook
|
||||
56
scripts/test-setup-sonobuoy-postgres
Normal file
56
scripts/test-setup-sonobuoy-postgres
Normal file
@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
|
||||
. ./scripts/test-setup-sonobuoy
|
||||
|
||||
# ---
|
||||
|
||||
cluster-pre-hook() {
|
||||
# gen-certs
|
||||
|
||||
mkdir -p $TEST_DIR/db/$LABEL_SUFFIX/metadata
|
||||
local testID=$(basename $TEST_DIR)
|
||||
local name=$(echo $LABEL_SUFFIX-$testID | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/name)
|
||||
local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/port)
|
||||
local secret=$(echo "${RANDOM}${RANDOM}${RANDOM}" | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/secret)
|
||||
|
||||
docker run --name $name \
|
||||
--privileged \
|
||||
-p 0.0.0.0:$port:5432 \
|
||||
-v $TEST_DIR/db-ca/:/db-ca \
|
||||
-e POSTGRES_USER=root \
|
||||
-e POSTGRES_PASSWORD=$secret \
|
||||
-d postgres:latest \
|
||||
>/dev/null
|
||||
# -c ssl=on \
|
||||
# -c ssl_ca_file=/db-ca/certs/ca.crt \
|
||||
# -c ssl_cert_file=/db-ca/certs/db.crt \
|
||||
# -c ssl_key_file=/db-ca/private/db.key \
|
||||
|
||||
local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/ip)
|
||||
# local host=host.docker.internal
|
||||
local host=172.17.0.1
|
||||
|
||||
DB_CONNECTION_TEST="
|
||||
docker run
|
||||
-v $TEST_DIR/db-ca/:/db-ca
|
||||
-e PGPASSWORD=$secret
|
||||
--rm postgres
|
||||
psql
|
||||
-h $host
|
||||
-p $port
|
||||
-U root
|
||||
-c \conninfo" \
|
||||
timeout --foreground 1m bash -c "wait-for-db-connection"
|
||||
# --set sslrootcert=/db-ca/certs/ca.crt
|
||||
# --set sslcert=/db-ca/certs/client.crt
|
||||
# --set sslkey=/db-ca/private/client.key
|
||||
|
||||
echo "Started $LABEL_SUFFIX db @ $host"
|
||||
export SERVER_ARGS="${SERVER_ARGS}
|
||||
--datastore-endpoint=postgres://root:$secret@$host:$port/testdb?sslmode=disable
|
||||
"
|
||||
# --datastore-cafile /db-ca/certs/ca.crt
|
||||
# --datastore-certfile /db-ca/certs/client.crt
|
||||
# --datastore-keyfile /db-ca/private/client.key
|
||||
}
|
||||
export -f cluster-pre-hook
|
||||
Loading…
Reference in New Issue
Block a user