From 168b14b08e016aed38160f00ecb3f71512f94a96 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 13 Jun 2022 13:32:13 -0700 Subject: [PATCH] Integration Test: Startup (#5630) * New startup integration test * Add testing section to PR template * Move helper functions to direct k8s client calls Signed-off-by: Derek Nola --- .github/PULL_REQUEST_TEMPLATE.md | 5 + .../certrotation/certrotation_int_test.go | 12 +- .../dualstack/dualstack_int_test.go | 6 +- .../etcdrestore/etcd_restore_int_test.go | 12 +- .../etcdsnapshot/etcdsnapshot_int_test.go | 6 +- tests/integration/integration.go | 91 ++++++++++++- .../localstorage/localstorage_int_test.go | 6 +- .../secretsencryption_int_test.go | 24 ++-- tests/integration/startup/startup_int_test.go | 123 ++++++++++++++++++ 9 files changed, 247 insertions(+), 38 deletions(-) create mode 100644 tests/integration/startup/startup_int_test.go diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 882fa40bce..85acea2a50 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,6 +13,11 @@ +#### Testing #### + + + + #### Linked Issues #### diff --git a/tests/integration/certrotation/certrotation_int_test.go b/tests/integration/certrotation/certrotation_int_test.go index c562f87d5c..a54f73dcb0 100644 --- a/tests/integration/certrotation/certrotation_int_test.go +++ b/tests/integration/certrotation/certrotation_int_test.go @@ -34,9 +34,9 @@ var _ = Describe("certificate rotation", func() { }) When("a new server is created", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get pods -A") - }, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) }) It("get certificate hash", func() { // get md5sum of the CA certs @@ -60,9 +60,9 @@ var _ = Describe("certificate rotation", func() { Expect(err).ToNot(HaveOccurred()) }) It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "360s", "5s").Should(Succeed()) }) It("get certificate hash", func() { // get md5sum of the CA certs diff --git a/tests/integration/dualstack/dualstack_int_test.go b/tests/integration/dualstack/dualstack_int_test.go index e4ba57038f..dc433eb7a2 100644 --- a/tests/integration/dualstack/dualstack_int_test.go +++ b/tests/integration/dualstack/dualstack_int_test.go @@ -39,9 +39,9 @@ var _ = Describe("dual stack", func() { }) When("a ipv4 and ipv6 cidr is present", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "180s", "5s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "10s").Should(Succeed()) }) It("creates pods with two IPs", func() { podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-n", "kube-system", "-o", "jsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}") diff --git a/tests/integration/etcdrestore/etcd_restore_int_test.go b/tests/integration/etcdrestore/etcd_restore_int_test.go index c106096266..f39e6f14f0 100644 --- a/tests/integration/etcdrestore/etcd_restore_int_test.go +++ b/tests/integration/etcdrestore/etcd_restore_int_test.go @@ -32,9 +32,9 @@ var _ = Describe("etcd snapshot restore", func() { }) When("a snapshot is restored on existing node", func() { It("etcd starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) }) It("create a workload", func() { result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl.yaml") @@ -79,9 +79,9 @@ var _ = Describe("etcd snapshot restore", func() { Expect(err).ToNot(HaveOccurred()) }) It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get", "pods", "-A") - }, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "360s", "5s").Should(Succeed()) }) It("Make sure Workload 1 exists", func() { Eventually(func() (string, error) { diff --git a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go index 3956032636..a6f0ec57d4 100644 --- a/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go +++ b/tests/integration/etcdsnapshot/etcdsnapshot_int_test.go @@ -33,9 +33,9 @@ var _ = Describe("etcd snapshots", func() { }) When("a new etcd is created", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl", "get pods -A") - }, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "10s").Should(Succeed()) }) It("saves an etcd snapshot", func() { Expect(testutil.K3sCmd("etcd-snapshot", "save")). diff --git a/tests/integration/integration.go b/tests/integration/integration.go index 030c628925..2989a0c4b0 100644 --- a/tests/integration/integration.go +++ b/tests/integration/integration.go @@ -3,6 +3,7 @@ package integration import ( "bufio" "bytes" + "context" "encoding/json" "fmt" "os" @@ -15,6 +16,10 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) // Compile-time variable @@ -120,6 +125,67 @@ func K3sServerArgs() []string { return args } +// K3sDefaultDeployments checks if the default deployments for K3s are ready, otherwise returns an error +func K3sDefaultDeployments() error { + return CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}) +} + +// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error +func CheckDeployments(deployments []string) error { + + deploymentSet := make(map[string]bool) + for _, d := range deployments { + deploymentSet[d] = false + } + + client, err := k8sClient() + if err != nil { + return err + } + deploymentList, err := client.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + return err + } + for _, deployment := range deploymentList.Items { + if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas { + deploymentSet[deployment.Name] = true + } + } + for d, found := range deploymentSet { + if !found { + return fmt.Errorf("failed to deploy %s", d) + } + } + + return nil +} + +func ParsePods() ([]corev1.Pod, error) { + clientSet, err := k8sClient() + if err != nil { + return nil, err + } + pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return pods.Items, nil +} + +func ParseNodes() ([]corev1.Node, error) { + clientSet, err := k8sClient() + if err != nil { + return nil, err + } + nodes, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return nodes.Items, nil +} + func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool { for scanner.Scan() { if strings.Contains(scanner.Text(), target) { @@ -158,7 +224,6 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) { } // K3sKillServer terminates the running K3s server and its children -// and unlocks the file for other tests func K3sKillServer(server *K3sServer) error { pgid, err := syscall.Getpgid(server.cmd.Process.Pid) if err != nil { @@ -180,9 +245,10 @@ func K3sKillServer(server *K3sServer) error { return nil } -// K3sCleanup attempts to cleanup networking and files leftover from an integration test -// this is similar to the k3s-killall.sh script, but we dynamically generate that on -// install, so we don't have access to it in testing. +// K3sCleanup unlocks the test-lock and +// attempts to cleanup networking and files leftover from an integration test. +// This is similar to the k3s-killall.sh script, but we dynamically generate that on +// install, so we don't have access to it during testing. func K3sCleanup(k3sTestLock int, dataDir string) error { if cni0Link, err := netlink.LinkByName("cni0"); err == nil { links, _ := netlink.LinkList() @@ -206,7 +272,10 @@ func K3sCleanup(k3sTestLock int, dataDir string) error { if err := os.RemoveAll(dataDir); err != nil { return err } - return flock.Release(k3sTestLock) + if k3sTestLock != -1 { + return flock.Release(k3sTestLock) + } + return nil } // RunCommand Runs command on the host @@ -220,3 +289,15 @@ func RunCommand(cmd string) (string, error) { } return out.String(), nil } + +func k8sClient() (*kubernetes.Clientset, error) { + config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/k3s/k3s.yaml") + if err != nil { + return nil, err + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return clientSet, nil +} diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go index b541c1836a..9504a6a334 100644 --- a/tests/integration/localstorage/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -34,9 +34,9 @@ var _ = Describe("local storage", func() { }) When("a new local storage is created", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "120s", "5s").Should(Succeed()) }) It("creates a new pvc", func() { result, err := testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pvc.yaml") diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go index 92aa101be2..18e8d38912 100644 --- a/tests/integration/secretsencryption/secretsencryption_int_test.go +++ b/tests/integration/secretsencryption/secretsencryption_int_test.go @@ -34,9 +34,9 @@ var _ = Describe("secrets encryption rotation", func() { }) When("A server starts with secrets encryption", func() { It("starts up with no problems", func() { - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) }) It("it creates a encryption key", func() { result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir) @@ -65,9 +65,9 @@ var _ = Describe("secrets encryption rotation", func() { Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) }) It("rotates the keys", func() { Eventually(func() (string, error) { @@ -89,9 +89,9 @@ var _ = Describe("secrets encryption rotation", func() { Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) time.Sleep(10 * time.Second) }) It("reencrypts the keys", func() { @@ -123,9 +123,9 @@ var _ = Describe("secrets encryption rotation", func() { Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) Expect(err).ToNot(HaveOccurred()) - Eventually(func() (string, error) { - return testutil.K3sCmd("kubectl get pods -A") - }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) time.Sleep(10 * time.Second) }) It("reencrypts the keys", func() { diff --git a/tests/integration/startup/startup_int_test.go b/tests/integration/startup/startup_int_test.go new file mode 100644 index 0000000000..64ca170bae --- /dev/null +++ b/tests/integration/startup/startup_int_test.go @@ -0,0 +1,123 @@ +package integration + +import ( + "testing" + + testutil "github.com/k3s-io/k3s/tests/integration" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" +) + +var startupServer *testutil.K3sServer +var startupServerArgs = []string{} +var testLock int + +var _ = BeforeSuite(func() { + if testutil.IsExistingServer() { + Skip("Test does not support running on existing k3s servers") + } + var err error + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) +}) + +var _ = Describe("startup tests", func() { + + When("a default server is created", func() { + It("is created with no arguments", func() { + var err error + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the default pods deployed", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "90s", "5s").Should(Succeed()) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + }) + }) + When("a etcd backed server is created", func() { + It("is created with cluster-init arguments", func() { + var err error + startupServerArgs = []string{"--cluster-init"} + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the default pods deployed", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "90s", "5s").Should(Succeed()) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + }) + }) + When("a server without traefik is created", func() { + It("is created with disable arguments", func() { + var err error + startupServerArgs = []string{"--disable", "traefik"} + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the default pods without traefik deployed", func() { + Eventually(func() error { + return testutil.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"}) + }, "90s", "10s").Should(Succeed()) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) + }) + }) + When("a server with different IPs is created", func() { + It("creates dummy interfaces", func() { + Expect(testutil.RunCommand("ip link add dummy2 type dummy")).To(Equal("")) + Expect(testutil.RunCommand("ip link add dummy3 type dummy")).To(Equal("")) + Expect(testutil.RunCommand("ip addr add 11.22.33.44/24 dev dummy2")).To(Equal("")) + Expect(testutil.RunCommand("ip addr add 55.66.77.88/24 dev dummy3")).To(Equal("")) + }) + It("is created with node-ip arguments", func() { + var err error + startupServerArgs = []string{"--node-ip", "11.22.33.44", "--node-external-ip", "55.66.77.88"} + startupServer, err = testutil.K3sStartServer(startupServerArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("has the node deployed with correct IPs", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "90s", "10s").Should(Succeed()) + + nodes, err := testutil.ParseNodes() + Expect(err).NotTo(HaveOccurred()) + Expect(nodes).To(HaveLen(1)) + Expect(nodes[0].Status.Addresses).To(ContainElements([]v1.NodeAddress{ + { + Type: "InternalIP", + Address: "11.22.33.44", + }, + { + Type: "ExternalIP", + Address: "55.66.77.88", + }})) + }) + It("dies cleanly", func() { + Expect(testutil.K3sKillServer(startupServer)).To(Succeed()) + Expect(testutil.RunCommand("ip link del dummy2")).To(Equal("")) + Expect(testutil.RunCommand("ip link del dummy3")).To(Equal("")) + }) + }) +}) + +var _ = AfterSuite(func() { + if !testutil.IsExistingServer() { + Expect(testutil.K3sCleanup(testLock, "")).To(Succeed()) + } +}) + +func Test_IntegrationStartup(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Startup Suite") +}