Integration Test: Startup (#5630)

* New startup integration test
* Add testing section to PR template
* Move helper functions to direct k8s client calls

Signed-off-by: Derek Nola <derek.nola@suse.com>

.github/PULL_REQUEST_TEMPLATE.md

@@ -13,6 +13,11 @@
 
 <!-- How can the changes be verified? Please provide whatever additional information necessary to help verify the proposed changes. -->
 
+#### Testing ####
+
+<!-- Is this change covered by testing? If not, consider adding a Unit or Integration test. -->
+<!-- See https://github.com/k3s-io/k3s/blob/master/tests/TESTING.md for more info -->
+
 #### Linked Issues ####
 
 <!-- Link any related issues, pull-requests, or commit hashes that are relevant to this pull request. If you are opening a PR without a corresponding issue please consider creating one first, at https://github.com/k3s-io/k3s/issues . A functional example will greatly help QA with verifying/reproducing a bug or testing new features. -->

tests/integration/certrotation/certrotation_int_test.go

@@ -34,9 +34,9 @@ var _ = Describe("certificate rotation", func() {
 	})
 	When("a new server is created", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get pods -A")
-			}, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("get certificate hash", func() {
 			// get md5sum of the CA certs
@@ -60,9 +60,9 @@ var _ = Describe("certificate rotation", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "360s", "5s").Should(Succeed())
 		})
 		It("get certificate hash", func() {
 			// get md5sum of the CA certs

tests/integration/dualstack/dualstack_int_test.go

@@ -39,9 +39,9 @@ var _ = Describe("dual stack", func() {
 	})
 	When("a ipv4 and ipv6 cidr is present", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "180s", "5s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "10s").Should(Succeed())
 		})
 		It("creates pods with two IPs", func() {
 			podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-n", "kube-system", "-o", "jsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}")

tests/integration/etcdrestore/etcd_restore_int_test.go

@@ -32,9 +32,9 @@ var _ = Describe("etcd snapshot restore", func() {
 	})
 	When("a snapshot is restored on existing node", func() {
 		It("etcd starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("create a workload", func() {
 			result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/temp_depl.yaml")
@@ -79,9 +79,9 @@ var _ = Describe("etcd snapshot restore", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
-			}, "360s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "360s", "5s").Should(Succeed())
 		})
 		It("Make sure Workload 1 exists", func() {
 			Eventually(func() (string, error) {

tests/integration/etcdsnapshot/etcdsnapshot_int_test.go

@@ -33,9 +33,9 @@ var _ = Describe("etcd snapshots", func() {
 	})
 	When("a new etcd is created", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get pods -A")
-			}, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "10s").Should(Succeed())
 		})
 		It("saves an etcd snapshot", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "save")).

tests/integration/integration.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -15,6 +16,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 // Compile-time variable
@@ -120,6 +125,67 @@ func K3sServerArgs() []string {
 	return args
 }
 
+// K3sDefaultDeployments checks if the default deployments for K3s are ready, otherwise returns an error
+func K3sDefaultDeployments() error {
+	return CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"})
+}
+
+// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error
+func CheckDeployments(deployments []string) error {
+
+	deploymentSet := make(map[string]bool)
+	for _, d := range deployments {
+		deploymentSet[d] = false
+	}
+
+	client, err := k8sClient()
+	if err != nil {
+		return err
+	}
+	deploymentList, err := client.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, deployment := range deploymentList.Items {
+		if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas {
+			deploymentSet[deployment.Name] = true
+		}
+	}
+	for d, found := range deploymentSet {
+		if !found {
+			return fmt.Errorf("failed to deploy %s", d)
+		}
+	}
+
+	return nil
+}
+
+func ParsePods() ([]corev1.Pod, error) {
+	clientSet, err := k8sClient()
+	if err != nil {
+		return nil, err
+	}
+	pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return pods.Items, nil
+}
+
+func ParseNodes() ([]corev1.Node, error) {
+	clientSet, err := k8sClient()
+	if err != nil {
+		return nil, err
+	}
+	nodes, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return nodes.Items, nil
+}
+
 func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool {
 	for scanner.Scan() {
 		if strings.Contains(scanner.Text(), target) {
@@ -158,7 +224,6 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
 }
 
 // K3sKillServer terminates the running K3s server and its children
-// and unlocks the file for other tests
 func K3sKillServer(server *K3sServer) error {
 	pgid, err := syscall.Getpgid(server.cmd.Process.Pid)
 	if err != nil {
@@ -180,9 +245,10 @@ func K3sKillServer(server *K3sServer) error {
 	return nil
 }
 
-// K3sCleanup attempts to cleanup networking and files leftover from an integration test
-// this is similar to the k3s-killall.sh script, but we dynamically generate that on
-// install, so we don't have access to it in testing.
+// K3sCleanup unlocks the test-lock and
+// attempts to cleanup networking and files leftover from an integration test.
+// This is similar to the k3s-killall.sh script, but we dynamically generate that on
+// install, so we don't have access to it during testing.
 func K3sCleanup(k3sTestLock int, dataDir string) error {
 	if cni0Link, err := netlink.LinkByName("cni0"); err == nil {
 		links, _ := netlink.LinkList()
@@ -206,7 +272,10 @@ func K3sCleanup(k3sTestLock int, dataDir string) error {
 	if err := os.RemoveAll(dataDir); err != nil {
 		return err
 	}
-	return flock.Release(k3sTestLock)
+	if k3sTestLock != -1 {
+		return flock.Release(k3sTestLock)
+	}
+	return nil
 }
 
 // RunCommand Runs command on the host
@@ -220,3 +289,15 @@ func RunCommand(cmd string) (string, error) {
 	}
 	return out.String(), nil
 }
+
+func k8sClient() (*kubernetes.Clientset, error) {
+	config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/k3s/k3s.yaml")
+	if err != nil {
+		return nil, err
+	}
+	clientSet, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return clientSet, nil
+}

tests/integration/localstorage/localstorage_int_test.go

@@ -34,9 +34,9 @@ var _ = Describe("local storage", func() {
 	})
 	When("a new local storage is created", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "120s", "5s").Should(Succeed())
 		})
 		It("creates a new pvc", func() {
 			result, err := testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pvc.yaml")

tests/integration/secretsencryption/secretsencryption_int_test.go

@@ -34,9 +34,9 @@ var _ = Describe("secrets encryption rotation", func() {
 	})
 	When("A server starts with secrets encryption", func() {
 		It("starts up with no problems", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("it creates a encryption key", func() {
 			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
@@ -65,9 +65,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 		})
 		It("rotates the keys", func() {
 			Eventually(func() (string, error) {
@@ -89,9 +89,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 			time.Sleep(10 * time.Second)
 		})
 		It("reencrypts the keys", func() {
@@ -123,9 +123,9 @@ var _ = Describe("secrets encryption rotation", func() {
 			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl get pods -A")
-			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
 			time.Sleep(10 * time.Second)
 		})
 		It("reencrypts the keys", func() {

tests/integration/startup/startup_int_test.go

@@ -0,0 +1,123 @@
+package integration
+
+import (
+	"testing"
+
+	testutil "github.com/k3s-io/k3s/tests/integration"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+)
+
+var startupServer *testutil.K3sServer
+var startupServerArgs = []string{}
+var testLock int
+
+var _ = BeforeSuite(func() {
+	if testutil.IsExistingServer() {
+		Skip("Test does not support running on existing k3s servers")
+	}
+	var err error
+	testLock, err = testutil.K3sTestLock()
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = Describe("startup tests", func() {
+
+	When("a default server is created", func() {
+		It("is created with no arguments", func() {
+			var err error
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "90s", "5s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+		})
+	})
+	When("a etcd backed server is created", func() {
+		It("is created with cluster-init arguments", func() {
+			var err error
+			startupServerArgs = []string{"--cluster-init"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "90s", "5s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+		})
+	})
+	When("a server without traefik is created", func() {
+		It("is created with disable arguments", func() {
+			var err error
+			startupServerArgs = []string{"--disable", "traefik"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods without traefik deployed", func() {
+			Eventually(func() error {
+				return testutil.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"})
+			}, "90s", "10s").Should(Succeed())
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
+		})
+	})
+	When("a server with different IPs is created", func() {
+		It("creates dummy interfaces", func() {
+			Expect(testutil.RunCommand("ip link add dummy2 type dummy")).To(Equal(""))
+			Expect(testutil.RunCommand("ip link add dummy3 type dummy")).To(Equal(""))
+			Expect(testutil.RunCommand("ip addr add 11.22.33.44/24 dev dummy2")).To(Equal(""))
+			Expect(testutil.RunCommand("ip addr add 55.66.77.88/24 dev dummy3")).To(Equal(""))
+		})
+		It("is created with node-ip arguments", func() {
+			var err error
+			startupServerArgs = []string{"--node-ip", "11.22.33.44", "--node-external-ip", "55.66.77.88"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the node deployed with correct IPs", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "90s", "10s").Should(Succeed())
+
+			nodes, err := testutil.ParseNodes()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(nodes).To(HaveLen(1))
+			Expect(nodes[0].Status.Addresses).To(ContainElements([]v1.NodeAddress{
+				{
+					Type:    "InternalIP",
+					Address: "11.22.33.44",
+				},
+				{
+					Type:    "ExternalIP",
+					Address: "55.66.77.88",
+				}}))
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.RunCommand("ip link del dummy2")).To(Equal(""))
+			Expect(testutil.RunCommand("ip link del dummy3")).To(Equal(""))
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	if !testutil.IsExistingServer() {
+		Expect(testutil.K3sCleanup(testLock, "")).To(Succeed())
+	}
+})
+
+func Test_IntegrationStartup(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Startup Suite")
+}