From 60297a1bbe33409ff67cf404ba5d2aa27091bc00 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 30 Aug 2021 08:27:59 -0700 Subject: [PATCH] Creation of K3s integration test Sonobuoy plugin (#3931) * Added test runner and build files * Changes to int test to output junit results. * Updated documentation, removed comments Signed-off-by: dereknola --- pkg/etcd/etcd_int_test.go | 5 +- pkg/flock/flock_int_test.go | 5 +- scripts/build-tests-sonobuoy | 66 ++++++++++++++++++++++ tests/TESTING.md | 16 +++++- tests/integration/Dockerfile.test | 16 ++++++ tests/integration/localstorage_int_test.go | 25 +++++--- tests/integration/test-runner.sh | 29 ++++++++++ 7 files changed, 149 insertions(+), 13 deletions(-) create mode 100755 scripts/build-tests-sonobuoy create mode 100644 tests/integration/Dockerfile.test create mode 100755 tests/integration/test-runner.sh diff --git a/pkg/etcd/etcd_int_test.go b/pkg/etcd/etcd_int_test.go index dba0b83005..3d7f17a034 100644 --- a/pkg/etcd/etcd_int_test.go +++ b/pkg/etcd/etcd_int_test.go @@ -7,6 +7,7 @@ import ( "time" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" testutil "github.com/rancher/k3s/tests/util" ) @@ -117,5 +118,7 @@ var _ = AfterSuite(func() { func Test_IntegrationEtcd(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Etcd Suite") + RunSpecsWithDefaultAndCustomReporters(t, "Etcd Suite", []Reporter{ + reporters.NewJUnitReporter("/tmp/results/junit-etcd.xml"), + }) } diff --git a/pkg/flock/flock_int_test.go b/pkg/flock/flock_int_test.go index 509cda6f38..2641d33880 100644 --- a/pkg/flock/flock_int_test.go +++ b/pkg/flock/flock_int_test.go @@ -4,6 +4,7 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "github.com/rancher/k3s/pkg/flock" ) @@ -30,5 +31,7 @@ var _ = Describe("file locks", func() { func TestFlock(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Flock Suite") + RunSpecsWithDefaultAndCustomReporters(t, "Flock Suite", []Reporter{ + reporters.NewJUnitReporter("/tmp/results/junit-flock.xml"), + }) } diff --git a/scripts/build-tests-sonobuoy b/scripts/build-tests-sonobuoy new file mode 100755 index 0000000000..3e0ff4e05a --- /dev/null +++ b/scripts/build-tests-sonobuoy @@ -0,0 +1,66 @@ +#!/bin/bash +set -e + +cd $(dirname $0)/.. + +REPO="k3s-int-tests" +OUTFILE="./dist/artifacts/k3s-int-tests.yaml" + +# Compile all integration tests and containerize them +mkdir -p dist/artifacts +go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-1.test ./tests/integration/... -run Integration + +PKG_TO_TEST=$(find ./pkg/ -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' |sort -u) +INDEX=1 +for i in $PKG_TO_TEST; do + echo $i + go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$INDEX.test $i -run Integration + INDEX=$(expr $INDEX + 1) +done +go test -c -v -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" -o dist/artifacts/k3s-integration-$INDEX.test ./tests/integration/... -run Integration +docker build -f ./tests/integration/Dockerfile.test -t $REPO . +docker save $REPO -o ./dist/artifacts/$REPO.tar + +sudo mkdir -p /var/lib/rancher/k3s/agent/images +sudo mv ./dist/artifacts/$REPO.tar /var/lib/rancher/k3s/agent/images/ + +# If k3s is already running, attempt to import the image +if [[ "$(pgrep k3s | wc -l)" -gt 0 ]]; then + sudo ./dist/artifacts/k3s ctr images import /var/lib/rancher/k3s/agent/images/$REPO.tar +fi + +# Cleanup compiled tests +rm dist/artifacts/k3s-integration-* + +# Generate the sonobuoy plugin and inject the necessary +# podSpec and volume mount modifications +PODSPEC=\ +' hostNetwork: true + volumes: + - name: var-k3s + hostPath: + path: /var/lib/rancher/k3s/ + type: Directory + - name: etc-k3s + hostPath: + path: /etc/rancher/k3s/ + type: Directory' +VOLMOUNTS=\ +' - mountPath: /var/lib/rancher/k3s/ + name: var-k3s + - mountPath: /etc/rancher/k3s/ + name: etc-k3s' + +sonobuoy gen plugin \ + --format=junit \ + --image ${REPO} \ + --show-default-podspec \ + --name k3s-int \ + --type job \ + --cmd ./test-runner.sh \ + --env KUBECONFIG=/etc/rancher/k3s/k3s.yaml \ + > $OUTFILE +awk -v PS="$PODSPEC" '/podSpec:/{print;print PS;next}1' $OUTFILE > ./dist/artifacts/temp.yaml +mv ./dist/artifacts/temp.yaml $OUTFILE +awk -v VM="$VOLMOUNTS" '/volumeMounts:/{print;print VM;next}1' $OUTFILE > ./dist/artifacts/temp.yaml +mv ./dist/artifacts/temp.yaml $OUTFILE diff --git a/tests/TESTING.md b/tests/TESTING.md index d5a17c3273..169e714929 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -76,11 +76,23 @@ Integration tests can be run with no k3s cluster present, each test will spin up go test ./pkg/... ./tests/... -run Integration ``` -Integration tests can also be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly. -``` +Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly. +```bash go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/... -run Integration ``` +Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster. +```bash +./scripts/build-tests-sonobuoy +sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy run --plugin ./dist/artifacts/k3s-int-tests.yaml +``` +Check the sonobuoy status and retrieve results +``` +sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy status +sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy retrieve +sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml sonobuoy results +``` + ___ ## End-to-End (E2E) Tests diff --git a/tests/integration/Dockerfile.test b/tests/integration/Dockerfile.test new file mode 100644 index 0000000000..7239a1ac90 --- /dev/null +++ b/tests/integration/Dockerfile.test @@ -0,0 +1,16 @@ +FROM golang:buster + + +RUN apt update && \ + apt install -y curl git lsof bash openssh-server gcc g++ make ca-certificates && \ + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + +WORKDIR $GOPATH/src/github.com/rancher/k3s-io/k3s/ + +COPY ./tests/testdata ./testdata +COPY ./tests/integration/test-runner.sh . +COPY ./dist/artifacts/k3s /usr/local/bin +COPY ./dist/artifacts/k3s-integration-* ./tests/ + +RUN go get -u github.com/onsi/gomega +RUN go get -u github.com/onsi/ginkgo diff --git a/tests/integration/localstorage_int_test.go b/tests/integration/localstorage_int_test.go index 1b664974b0..ae3df12856 100644 --- a/tests/integration/localstorage_int_test.go +++ b/tests/integration/localstorage_int_test.go @@ -8,6 +8,7 @@ import ( "testing" . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" testutil "github.com/rancher/k3s/tests/util" ) @@ -35,8 +36,9 @@ var _ = Describe("local storage", func() { }, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) }) It("creates a new pvc", func() { - Expect(testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pvc.yaml")). - To(ContainSubstring("persistentvolumeclaim/local-path-pvc created")) + result, err := testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pvc.yaml") + Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created")) + Expect(err).NotTo(HaveOccurred()) }) It("creates a new pod", func() { Expect(testutil.K3sCmd("kubectl", "create", "-f", "../testdata/localstorage_pod.yaml")). @@ -44,11 +46,14 @@ var _ = Describe("local storage", func() { }) It("shows storage up in kubectl", func() { Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pvc") + return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pvc") }, "45s", "1s").Should(MatchRegexp(`local-path-pvc.+Bound`)) Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pv") + return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv") }, "10s", "1s").Should(MatchRegexp(`pvc.+2Gi.+Bound`)) + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pod") + }, "10s", "1s").Should(MatchRegexp(`volume-test.+Running`)) }) It("has proper folder permissions", func() { var k3sStorage = "/var/lib/rancher/k3s/storage" @@ -56,7 +61,7 @@ var _ = Describe("local storage", func() { Expect(err).ToNot(HaveOccurred()) Expect(fmt.Sprintf("%04o", fileStat.Mode().Perm())).To(Equal("0701")) - pvResult, err := testutil.K3sCmd("kubectl", "get", "pv") + pvResult, err := testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv") Expect(err).ToNot(HaveOccurred()) reg, err := regexp.Compile(`pvc[^\s]+`) Expect(err).ToNot(HaveOccurred()) @@ -66,9 +71,9 @@ var _ = Describe("local storage", func() { Expect(fmt.Sprintf("%04o", fileStat.Mode().Perm())).To(Equal("0777")) }) It("deletes properly", func() { - Expect(testutil.K3sCmd("kubectl", "delete", "pod", "volume-test")). - To(ContainSubstring("pod \"volume-test\" deleted")) - Expect(testutil.K3sCmd("kubectl", "delete", "pvc", "local-path-pvc")). + Expect(testutil.K3sCmd("kubectl", "delete", "--namespace=default", "--force", "pod", "volume-test")). + To(ContainSubstring("pod \"volume-test\" force deleted")) + Expect(testutil.K3sCmd("kubectl", "delete", "--namespace=default", "pvc", "local-path-pvc")). To(ContainSubstring("persistentvolumeclaim \"local-path-pvc\" deleted")) }) }) @@ -82,5 +87,7 @@ var _ = AfterSuite(func() { func Test_IntegrationLocalStorage(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Local Storage Suite") + RunSpecsWithDefaultAndCustomReporters(t, "Local Storage Suite", []Reporter{ + reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"), + }) } diff --git a/tests/integration/test-runner.sh b/tests/integration/test-runner.sh new file mode 100755 index 0000000000..752b5af96b --- /dev/null +++ b/tests/integration/test-runner.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -x + +results_dir="${RESULTS_DIR:-/tmp/results}" + +# saveResults prepares the results for handoff to the Sonobuoy worker. +# See: https://github.com/vmware-tanzu/sonobuoy/blob/master/site/content/docs/master/plugins.md +saveResults() { + cd ${results_dir} + + # Sonobuoy worker expects a tar file. + tar czf results.tar.gz * + + # Signal to the worker that we are done and where to find the results. + printf ${results_dir}/results.tar.gz > ${results_dir}/done +} + +# Ensure that we tell the Sonobuoy worker we are done regardless of results. +trap saveResults EXIT + +runTests() { + cd ./tests + for t in *.test; do + # Run each test (automatically saves the output in the results directory). + ./$t + done +} +runTests