mirror of https://github.com/k3s-io/k3s.git
Upload logs to gcloud
This commit is contained in:
Erik Wilson
parent
e6f8418820
commit
c62c428204
12
.drone.yml
12
.drone.yml
|
|
@ -57,6 +57,10 @@ steps:
|
|||
|
||||
- name: sonobuoy-e2e-tests
|
||||
image: rancher/dapper:v0.4.2
|
||||
secrets: [ gcloud_auth ]
|
||||
environment:
|
||||
GCLOUD_AUTH:
|
||||
from_secret: gcloud_auth
|
||||
commands:
|
||||
- dapper -f Dockerfile.sonobuoy.dapper
|
||||
volumes:
|
||||
|
|
@ -144,6 +148,10 @@ steps:
|
|||
|
||||
- name: sonobuoy-e2e-tests
|
||||
image: rancher/dapper:v0.4.2
|
||||
secrets: [ gcloud_auth ]
|
||||
environment:
|
||||
GCLOUD_AUTH:
|
||||
from_secret: gcloud_auth
|
||||
commands:
|
||||
- dapper -f Dockerfile.sonobuoy.dapper
|
||||
volumes:
|
||||
|
|
@ -214,6 +222,10 @@ steps:
|
|||
|
||||
- name: sonobuoy-e2e-tests
|
||||
image: rancher/dapper:v0.4.2
|
||||
secrets: [ gcloud_auth ]
|
||||
environment:
|
||||
GCLOUD_AUTH:
|
||||
from_secret: gcloud_auth
|
||||
commands:
|
||||
- dapper -f Dockerfile.sonobuoy.dapper
|
||||
volumes:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
FROM golang:1.13.3-alpine3.10
|
||||
|
||||
RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils
|
||||
RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2
|
||||
RUN SONOBUOY_VER=v0.16.2 && \
|
||||
SONOBUOY_PKG=github.com/vmware-tanzu/sonobuoy && \
|
||||
go get -d ${SONOBUOY_PKG} && \
|
||||
|
|
@ -18,7 +18,7 @@ RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \
|
|||
chmod a+x /usr/local/bin/kubectl
|
||||
|
||||
ENV DAPPER_RUN_ARGS --privileged --network host
|
||||
ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME
|
||||
ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME GCLOUD_AUTH
|
||||
ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
|
||||
ENV DAPPER_OUTPUT ./dist
|
||||
ENV DAPPER_DOCKER_SOCKET true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -d "$1" ] || {
|
||||
echo "First argument should be a directory" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
umask 077
|
||||
|
||||
TMPDIR=$(mktemp -d)
|
||||
cleanup() {
|
||||
exit_code=$?
|
||||
trap - EXIT INT
|
||||
rm -rf ${TMPDIR}
|
||||
exit ${exit_code}
|
||||
}
|
||||
trap cleanup EXIT INT
|
||||
|
||||
GCLOUD_JSON=${TMPDIR}/.gcloud.json
|
||||
[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON}
|
||||
[ -s "${GCLOUD_JSON}" ] || {
|
||||
echo "gcloud auth not defined" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
BOTO_CONF=${TMPDIR}/.boto
|
||||
[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <<END
|
||||
[Credentials]
|
||||
gs_service_key_file = ${GCLOUD_JSON}
|
||||
[Boto]
|
||||
https_validate_certificates = True
|
||||
[GSUtil]
|
||||
content_language = en
|
||||
default_api_version = 2
|
||||
default_project_id = rancher-dev
|
||||
END
|
||||
|
||||
[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR}
|
||||
|
||||
HOME=${TMPDIR}
|
||||
PATH=$PATH:${HOME}/gsutil
|
||||
|
||||
LOG_TGZ=k3s-log-$(uname -s | tr '[:upper:]' '[:lower:]' )-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(date +%s)-$(basename $1).tgz
|
||||
|
||||
tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1)
|
||||
gsutil cp ${TMPDIR}/${LOG_TGZ} gs://k3s-ci-logs || exit 1
|
||||
echo "Logs uploaded" >&2
|
||||
echo "https://storage.googleapis.com/k3s-ci-logs/${LOG_TGZ}"
|
||||
153
scripts/sonobuoy
153
scripts/sonobuoy
|
|
@ -1,9 +1,10 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
cd $(dirname $0)/..
|
||||
|
||||
. ./scripts/test-helpers
|
||||
|
||||
if [ -z "${K3S_IMAGE}" ]; then
|
||||
echo 'K3S_IMAGE environment variable should be defined'
|
||||
exit 1
|
||||
|
|
@ -11,150 +12,6 @@ fi
|
|||
|
||||
# ---
|
||||
|
||||
port-used() {
|
||||
(cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
|
||||
}
|
||||
export -f port-used
|
||||
|
||||
# ---
|
||||
|
||||
get-port() {
|
||||
while
|
||||
PORT=$((10000 + RANDOM % 50000))
|
||||
port-used ${PORT}
|
||||
do continue; done
|
||||
echo ${PORT}
|
||||
}
|
||||
export -f get-port
|
||||
|
||||
# ---
|
||||
|
||||
fetch-kubeconfig() {
|
||||
docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
|
||||
}
|
||||
export -f fetch-kubeconfig
|
||||
|
||||
# ---
|
||||
|
||||
wait-for-kubeconfig() {
|
||||
while ! fetch-kubeconfig; do
|
||||
echo 'Waiting for kubeconfig to become available...'
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
export -f wait-for-kubeconfig
|
||||
|
||||
# ---
|
||||
|
||||
count-ready-nodes() {
|
||||
kubectl get nodes -o json \
|
||||
| jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
|
||||
| wc -l \
|
||||
| tr -d '[:space:]'
|
||||
}
|
||||
export -f count-ready-nodes
|
||||
|
||||
# ---
|
||||
|
||||
wait-for-nodes() {
|
||||
while [[ $(count-ready-nodes) -ne $1 ]]; do
|
||||
echo 'Waiting for nodes to be ready...'
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
export -f wait-for-nodes
|
||||
|
||||
# ---
|
||||
|
||||
pod-ready() {
|
||||
kubectl get pods -n kube-system -o json \
|
||||
| jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
|
||||
}
|
||||
export -f pod-ready
|
||||
|
||||
# ---
|
||||
|
||||
wait-for-services() {
|
||||
for service in $@; do
|
||||
while [[ "$(pod-ready ${service})" != 'true' ]]; do
|
||||
echo "Waiting for service ${service} to be ready..."
|
||||
sleep 5
|
||||
done
|
||||
echo "Service ${service} is ready"
|
||||
done
|
||||
}
|
||||
export -f wait-for-services
|
||||
|
||||
# ---
|
||||
|
||||
verify-valid-version() {
|
||||
if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
export -f verify-valid-version
|
||||
|
||||
# ---
|
||||
|
||||
verify-valid-versions() {
|
||||
verify-valid-version $1 kubectl version
|
||||
verify-valid-version $1 ctr version
|
||||
verify-valid-version $1 crictl version
|
||||
}
|
||||
export -f verify-valid-versions
|
||||
|
||||
# ---
|
||||
|
||||
dump-container-logs() {
|
||||
mkdir -p ${LOGS}
|
||||
for container in ${CONTAINERS}; do
|
||||
docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/containerd-${container}.log
|
||||
docker logs ${container} >${LOGS}/${container}.log 2>&1
|
||||
done
|
||||
}
|
||||
export -f dump-container-logs
|
||||
|
||||
# ---
|
||||
|
||||
retrieve-sonobuoy-logs() {
|
||||
if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
|
||||
status=passed
|
||||
exit_code=0
|
||||
else
|
||||
status=failed
|
||||
exit_code=1
|
||||
fi
|
||||
|
||||
mkdir -p ${E2E}
|
||||
sonobuoy retrieve ${E2E}
|
||||
tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
|
||||
if [ ! -s ${RESULTS} ]; then
|
||||
return 1
|
||||
fi
|
||||
if [ -n "${E2E_LOG_OUTPUT}" ]; then
|
||||
cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
|
||||
fi
|
||||
awk '/^Summarizing .* Failures:$/,0' ${RESULTS}
|
||||
return ${exit_code}
|
||||
}
|
||||
export -f retrieve-sonobuoy-logs
|
||||
|
||||
# ---
|
||||
|
||||
sonobuoy-test() {
|
||||
time sonobuoy run \
|
||||
--config=scripts/sonobuoy-config.json \
|
||||
--plugin-env=e2e.E2E_USE_GO_RUNNER=true \
|
||||
--wait=30 \
|
||||
"${@}" &
|
||||
SONOBUOY_PID=$!
|
||||
wait $SONOBUOY_PID
|
||||
retrieve-sonobuoy-logs
|
||||
}
|
||||
export -f sonobuoy-test
|
||||
|
||||
# ---
|
||||
|
||||
cleanup() {
|
||||
exit_status=$?
|
||||
set +e
|
||||
|
|
@ -162,7 +19,7 @@ cleanup() {
|
|||
trap - EXIT
|
||||
[ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null
|
||||
if [ "${exit_status}" -ne "0" ]; then
|
||||
dump-container-logs
|
||||
dump-logs
|
||||
fi
|
||||
docker rm -f ${CONTAINERS} 2>/dev/null
|
||||
rm ${KUBECONFIG}
|
||||
|
|
@ -180,8 +37,8 @@ E2E_LOG='plugins/e2e/results/global/e2e.log'
|
|||
RESULTS="${E2E}/${E2E_LOG}"
|
||||
|
||||
SECRET=random-$((100000 + RANDOM % 999999))
|
||||
export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
|
||||
export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
|
||||
export K3S_AGENT=k3s-agent-${K3S_PORT}
|
||||
export K3S_SERVER=k3s-server-${K3S_PORT}
|
||||
export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
|
||||
export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
|
||||
|
||||
|
|
|
|||
|
|
@ -25,12 +25,12 @@ show-logs() {
|
|||
fi
|
||||
echo
|
||||
echo "#- Begin: logs for sonobuoy run pid ${pid}"
|
||||
for log in $(pwd)/logs/${pid}/*; do
|
||||
for log in $(pwd)/logs/${pid}/*.log; do
|
||||
if [ -f ${log} ]; then
|
||||
echo
|
||||
echo "#- Start: ${log}"
|
||||
cat ${log}
|
||||
echo "#- End: ${log}"
|
||||
echo "#- Tail: ${log}"
|
||||
tail -10 ${log}
|
||||
echo "#- Done: ${log}"
|
||||
echo
|
||||
fi
|
||||
done
|
||||
|
|
@ -57,18 +57,23 @@ cleanup() {
|
|||
}
|
||||
trap cleanup EXIT INT
|
||||
|
||||
|
||||
# ---
|
||||
|
||||
run-sonobuoy() {
|
||||
output+=(${log_output})
|
||||
E2E_LOG_OUTPUT=${log_output} ./scripts/sonobuoy ${@} &
|
||||
E2E_LOG_OUTPUT=${log_output} \
|
||||
./scripts/sonobuoy ${@} \
|
||||
> >(stdbuf -oL sed "s/^/[${label}] /") 2>&1 &
|
||||
pids+=($!)
|
||||
}
|
||||
|
||||
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log \
|
||||
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log label=PARALLEL \
|
||||
run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y
|
||||
|
||||
sleep 60
|
||||
|
||||
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log \
|
||||
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log label=SERIAL \
|
||||
run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
|
||||
|
||||
exit_code=0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
#!/bin/bash
|
||||
|
||||
# ---
|
||||
|
||||
port-used() {
|
||||
(cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
|
||||
}
|
||||
export -f port-used
|
||||
|
||||
# ---
|
||||
|
||||
get-port() {
|
||||
while
|
||||
PORT=$((10000 + RANDOM % 50000))
|
||||
port-used ${PORT}
|
||||
do continue; done
|
||||
echo ${PORT}
|
||||
}
|
||||
export -f get-port
|
||||
|
||||
# ---
|
||||
|
||||
fetch-kubeconfig() {
|
||||
docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
|
||||
}
|
||||
export -f fetch-kubeconfig
|
||||
|
||||
# ---
|
||||
|
||||
wait-for-kubeconfig() {
|
||||
while ! fetch-kubeconfig; do
|
||||
echo 'Waiting for kubeconfig to become available...'
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
export -f wait-for-kubeconfig
|
||||
|
||||
# ---
|
||||
|
||||
count-ready-nodes() {
|
||||
kubectl get nodes -o json \
|
||||
| jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
|
||||
| wc -l \
|
||||
| tr -d '[:space:]'
|
||||
}
|
||||
export -f count-ready-nodes
|
||||
|
||||
# ---
|
||||
|
||||
wait-for-nodes() {
|
||||
while [[ $(count-ready-nodes) -ne $1 ]]; do
|
||||
echo 'Waiting for nodes to be ready...'
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
export -f wait-for-nodes
|
||||
|
||||
# ---
|
||||
|
||||
pod-ready() {
|
||||
kubectl get pods -n kube-system -o json \
|
||||
| jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
|
||||
}
|
||||
export -f pod-ready
|
||||
|
||||
# ---
|
||||
|
||||
wait-for-services() {
|
||||
for service in $@; do
|
||||
while [[ "$(pod-ready ${service})" != 'true' ]]; do
|
||||
echo "Waiting for service ${service} to be ready..."
|
||||
sleep 5
|
||||
done
|
||||
echo "Service ${service} is ready"
|
||||
done
|
||||
}
|
||||
export -f wait-for-services
|
||||
|
||||
# ---
|
||||
|
||||
verify-valid-version() {
|
||||
if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
export -f verify-valid-version
|
||||
|
||||
# ---
|
||||
|
||||
verify-valid-versions() {
|
||||
verify-valid-version $1 kubectl version
|
||||
verify-valid-version $1 ctr version
|
||||
verify-valid-version $1 crictl version
|
||||
}
|
||||
export -f verify-valid-versions
|
||||
|
||||
# ---
|
||||
|
||||
dump-logs() {
|
||||
mkdir -p ${LOGS}/sonobuoy
|
||||
tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${LOGS}/sonobuoy
|
||||
for container in ${CONTAINERS}; do
|
||||
docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/${container}-containerd.log
|
||||
docker logs ${container} >${LOGS}/${container}-system.log 2>&1
|
||||
done
|
||||
./scripts/log-upload ${LOGS}
|
||||
}
|
||||
export -f dump-logs
|
||||
|
||||
# ---
|
||||
|
||||
retrieve-sonobuoy-logs() {
|
||||
sonobuoy status || true
|
||||
if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
|
||||
status=passed
|
||||
exit_code=0
|
||||
else
|
||||
status=failed
|
||||
exit_code=1
|
||||
fi
|
||||
|
||||
mkdir -p ${E2E}
|
||||
sonobuoy retrieve ${E2E}
|
||||
tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
|
||||
if [ ! -s ${RESULTS} ]; then
|
||||
return 1
|
||||
fi
|
||||
if [ -n "${E2E_LOG_OUTPUT}" ]; then
|
||||
cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
|
||||
fi
|
||||
awk '/^Summarizing .* Failures:$/,0' ${RESULTS}
|
||||
return ${exit_code}
|
||||
}
|
||||
export -f retrieve-sonobuoy-logs
|
||||
|
||||
# ---
|
||||
|
||||
sonobuoy-test() {
|
||||
time sonobuoy run \
|
||||
--config=scripts/sonobuoy-config.json \
|
||||
--plugin-env=e2e.E2E_USE_GO_RUNNER=true \
|
||||
--wait=30 \
|
||||
"${@}" &
|
||||
SONOBUOY_PID=$!
|
||||
wait $SONOBUOY_PID
|
||||
retrieve-sonobuoy-logs
|
||||
}
|
||||
export -f sonobuoy-test
|
||||
|
||||
# ---
|
||||
Loading…
Reference in New Issue