From 7175ebe2be3bc307ed45c0a0909aa03e479cde53 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 5 May 2023 11:00:47 -0700 Subject: [PATCH] E2E: Startup test cleanup + RunCommand Enhancement (#7388) * Add beforesuite to startup * Reduce timeouts for startup * Fix cleanup + set kubeconfig Signed-off-by: Derek Nola --- .drone.yml | 1 + tests/e2e/startup/startup_test.go | 44 ++++++++++++++++++------------- tests/e2e/testutils.go | 11 +++++--- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/.drone.yml b/.drone.yml index 73ef977f02..1395d72b24 100644 --- a/.drone.yml +++ b/.drone.yml @@ -752,6 +752,7 @@ steps: - | if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then cd ../upgradecluster + vagrant destroy -f E2E_RELEASE_CHANNEL="latest" go test -v -timeout=45m ./upgradecluster_test.go -ci -local fi - docker stop registry && docker rm registry diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index f33fedb026..6c3858f195 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -71,23 +71,29 @@ func KillK3sCluster(nodes []string) error { if _, err := e2e.RunCmdOnNode("sudo k3s-killall.sh", node); err != nil { return err } + if strings.Contains(node, "server") { + if _, err := e2e.RunCmdOnNode("sudo rm -rf /var/lib/rancher/k3s/server/db", node); err != nil { + return err + } + } } return nil } var _ = ReportAfterEach(e2e.GenReport) +var _ = BeforeSuite(func() { + var err error + if *local { + serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, 1, 1) + } else { + serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, 1, 1) + } + Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) +}) + var _ = Describe("Various Startup Configurations", Ordered, func() { Context("Verify CRI-Dockerd :", func() { - It("Stands up the nodes", func() { - var err error - if *local { - serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, 1, 1) - } else { - serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, 1, 1) - } - Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) - }) It("Starts K3s with no issues", func() { dockerYAML := "docker: true" err := StartK3sCluster(append(serverNodeNames, agentNodeNames...), dockerYAML, dockerYAML) @@ -109,7 +115,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParseNodes(kubeConfigFile, true) fmt.Printf("\nFetching pods status\n") @@ -123,7 +129,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { g.Expect(pod.Status).Should(Equal("Running"), pod.Name) } } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParsePods(kubeConfigFile, true) }) It("Kills the cluster", func() { @@ -153,7 +159,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParseNodes(kubeConfigFile, true) fmt.Printf("\nFetching pods status\n") @@ -167,7 +173,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { g.Expect(pod.Status).Should(Equal("Running"), pod.Name) } } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParsePods(kubeConfigFile, true) }) It("Kills the cluster", func() { @@ -197,7 +203,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { for _, node := range nodes { g.Expect(node.Status).Should(Equal("Ready")) } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParseNodes(kubeConfigFile, true) fmt.Printf("\nFetching pods status\n") @@ -211,21 +217,21 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { g.Expect(pod.Status).Should(Equal("Running"), pod.Name) } } - }, "620s", "5s").Should(Succeed()) + }, "360s", "5s").Should(Succeed()) _, _ = e2e.ParsePods(kubeConfigFile, true) }) It("Returns pod metrics", func() { cmd := "kubectl top pod -A" Eventually(func() error { - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + _, err := e2e.RunCommand(cmd) return err - }, "620s", "5s").Should(Succeed()) + }, "600s", "5s").Should(Succeed()) }) It("Returns node metrics", func() { cmd := "kubectl top node" - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + _, err := e2e.RunCommand(cmd) Expect(err).NotTo(HaveOccurred()) }) @@ -237,7 +243,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { It("Collects logs from a pod", func() { cmd := "kubectl logs -n kube-system -l app.kubernetes.io/name=traefik -c traefik" - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + _, err := e2e.RunCommand(cmd) Expect(err).NotTo(HaveOccurred()) }) diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 484b06ad7f..16325bac83 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -303,6 +303,9 @@ func GenKubeConfigFile(serverName string) (string, error) { if err := os.WriteFile(kubeConfigFile, []byte(kubeConfig), 0644); err != nil { return "", err } + if err := os.Setenv("E2E_KUBECONFIG", kubeConfigFile); err != nil { + return "", err + } return kubeConfigFile, nil } @@ -437,17 +440,19 @@ func RestartClusterAgent(nodeNames []string) error { // RunCmdOnNode executes a command from within the given node func RunCmdOnNode(cmd string, nodename string) (string, error) { - runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename + runcmd := "vagrant ssh " + nodename + " -c \"" + cmd + "\"" out, err := RunCommand(runcmd) if err != nil { - return out, fmt.Errorf("failed to run command %s on node %s: %v", cmd, nodename, err) + return out, fmt.Errorf("failed to run command: %s on node %s: %s, %v", cmd, nodename, out, err) } return out, nil } -// RunCommand executes a command on the host func RunCommand(cmd string) (string, error) { c := exec.Command("bash", "-c", cmd) + if kc, ok := os.LookupEnv("E2E_KUBECONFIG"); ok { + c.Env = append(os.Environ(), "KUBECONFIG="+kc) + } out, err := c.CombinedOutput() return string(out), err }