Update terraform package and make running locally easier

Signed-off-by: rancher-max <max.ross@suse.com>

tests/terraform/README.md

@@ -0,0 +1,43 @@
+# Terraform (TF) Tests
+
+Terraform (TF) tests are an additional form of End-to-End (E2E) tests that cover multi-node K3s configuration and administration: install, update, teardown, etc. across a wide range of operating systems. Terraform tests are used as part of K3s quality assurance (QA) to bring up clusters with different configurations on demand, perform specific functionality tests, and keep them up and running to perform some exploratory tests in real-world scenarios.
+
+## Framework 
+TF tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the e2e tests. They rely on [Terraform](https://www.terraform.io/) to provide the underlying cluster configuration. 
+
+## Format
+
+- All TF tests should be placed under `tests/terraform/<TEST_NAME>`.
+- All TF test functions should be named: `Test_TF<TEST_NAME>`. 
+
+See the [create cluster test](../tests/terraform/createcluster_test.go) as an example.
+
+## Running
+
+Before running the tests, it's best to create a tfvars file in `./tests/terraform/modules/k3scluster/config/local.tfvars`. There is some information there to get you started, but the empty variables should be filled in appropriately per your AWS environment.
+
+All TF tests can be run with:
+```bash
+go test -timeout=60m ./tests/terrfaorm/... -run TF
+```
+Tests can be run individually with:
+```bash
+go test -timeout=30m ./tests/terraform/createcluster.go ./tests/terraform/createcluster_test.go ./tests/terraform/testutils.go
+# example with vars:
+go test -timeout=30m -v ./tests/terraform/createcluster.go ./tests/terraform/createcluster_test.go ./tests/terraform/testutils.go -node_os=ubuntu -aws_ami=ami-02f3416038bdb17fb -cluster_type=etcd -resource_name=localrun1 -sshuser=ubuntu -sshkey="key-name" -destroy=false
+```
+
+In between tests, if the cluster is not destroyed, then make sure to delete the ./tests/terraform/terraform.tfstate file if you want to create a new cluster.
+
+Additionally, to generate junit reporting for the tests, the Ginkgo CLI is used. Installation instructions can be found [here.](https://onsi.github.io/ginkgo/#getting-started)  
+
+To run the all TF tests and generate JUnit testing reports:
+```
+ginkgo --junit-report=result.xml ./tests/terraform/...
+```
+
+Note: The `go test` default timeout is 10 minutes, thus the `-timeout` flag should be used. The `ginkgo` default timeout is 1 hour, no timeout flag is needed.
+
+# Debugging
+The cluster and VMs can be retained after a test by passing `-destroy=false`. 
+To focus individual runs on specific test clauses, you can prefix with `F`. For example, in the [create cluster test](../tests/terraform/createcluster_test.go), you can upate the initial creation to be: `FIt("Starts up with no issues", func() {` in order to focus the run on only that clause.

tests/terraform/createcluster.go

@@ -1,4 +1,4 @@
-package e2e
+package terraform
 
 import (
 	"flag"
@@ -11,13 +11,18 @@ import (
 )
 
 var destroy = flag.Bool("destroy", false, "a bool")
-var nodeOs = flag.String("node_os", "centos8", "a string")
+var awsAmi = flag.String("aws_ami", "", "a valid ami string like ami-abcxyz123")
+var nodeOs = flag.String("node_os", "ubuntu", "a string")
 var externalDb = flag.String("external_db", "mysql", "a string")
 var arch = flag.String("arch", "amd64", "a string")
 var clusterType = flag.String("cluster_type", "etcd", "a string")
 var resourceName = flag.String("resource_name", "etcd", "a string")
 var sshuser = flag.String("sshuser", "ubuntu", "a string")
 var sshkey = flag.String("sshkey", "", "a string")
+var access_key = flag.String("access_key", "", "local path to the private sshkey")
+var tfVars = flag.String("tfvars", "./modules/k3scluster/config/local.tfvars", "custom .tfvars file")
+var serverNodes = flag.Int("no_of_server_nodes", 2, "count of server nodes")
+var workerNodes = flag.Int("no_of_worker_nodes", 1, "count of worker nodes")
 var failed = false
 
 var (
@@ -26,43 +31,44 @@ var (
 	workerIPs      string
 )
 
-func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool, arch string) (string, string, string, error) {
+func BuildCluster(nodeOs, awsAmi string, clusterType, externalDb, resourceName string, t *testing.T, destroy bool, arch string) (string, error) {
 	tDir := "./modules/k3scluster"
-	vDir := "/config/" + nodeOs + clusterType + ".tfvars"
-
-	if externalDb != "" {
-		vDir = "/config/" + nodeOs + externalDb + ".tfvars"
-	}
 
 	tfDir, err := filepath.Abs(tDir)
 	if err != nil {
-		return "", "", "", err
+		return "", err
 	}
-	varDir, err := filepath.Abs(vDir)
+	varDir, err := filepath.Abs(*tfVars)
 	if err != nil {
-		return "", "", "", err
+		return "", err
 	}
 	TerraformOptions := &terraform.Options{
 		TerraformDir: tfDir,
 		VarFiles:     []string{varDir},
 		Vars: map[string]interface{}{
-			"cluster_type":  clusterType,
-			"resource_name": resourceName,
-			"external_db":   externalDb,
+			"node_os":            nodeOs,
+			"aws_ami":            awsAmi,
+			"cluster_type":       clusterType,
+			"resource_name":      resourceName,
+			"external_db":        externalDb,
+			"aws_user":           *sshuser,
+			"key_name":           *sshkey,
+			"access_key":         *access_key,
+			"no_of_server_nodes": *serverNodes,
+			"no_of_worker_nodes": *workerNodes,
 		},
 	}
 
 	if destroy {
 		fmt.Printf("Cluster is being deleted")
 		terraform.Destroy(t, TerraformOptions)
-		return "", "", "", err
+		return "cluster destroyed", err
 	}
 
 	fmt.Printf("Creating Cluster")
 	terraform.InitAndApply(t, TerraformOptions)
-	kubeconfig := terraform.Output(t, TerraformOptions, "kubeconfig") + "_kubeconfig"
-	masterIPs := terraform.Output(t, TerraformOptions, "master_ips")
-	workerIPs := terraform.Output(t, TerraformOptions, "worker_ips")
-	kubeconfigFile := "/config/" + kubeconfig
-	return kubeconfigFile, masterIPs, workerIPs, err
+	kubeConfigFile = "/tmp/" + terraform.Output(t, TerraformOptions, "kubeconfig") + "_kubeconfig"
+	masterIPs = terraform.Output(t, TerraformOptions, "master_ips")
+	workerIPs = terraform.Output(t, TerraformOptions, "worker_ips")
+	return "cluster created", err
 }

tests/terraform/createcluster_test.go

@@ -1,17 +1,16 @@
-package e2e
+package terraform
 
 import (
 	"flag"
 	"fmt"
 	"strings"
 	"testing"
-	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
-func Test_E2EClusterCreateValidation(t *testing.T) {
+func Test_TFClusterCreateValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
 
@@ -21,10 +20,15 @@ func Test_E2EClusterCreateValidation(t *testing.T) {
 var _ = Describe("Test:", func() {
 	Context("Build Cluster:", func() {
 		It("Starts up with no issues", func() {
-			kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
+			status, err := BuildCluster(*nodeOs, *awsAmi, *clusterType, *externalDb, *resourceName, &testing.T{}, false, *arch)
 			Expect(err).NotTo(HaveOccurred())
+			Expect(status).To(Equal("cluster created"))
 			defer GinkgoRecover()
-			fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "BACKEND", *clusterType, *externalDb)
+			if strings.Contains(*clusterType, "etcd") {
+				fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "\nBACKEND", *clusterType)
+			} else {
+				fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "\nBACKEND", *externalDb)
+			}
 			fmt.Printf("\nIPs:\n")
 			fmt.Println("Server Node IPS:", masterIPs)
 			fmt.Println("Agent Node IPS:", workerIPs)
@@ -33,17 +37,29 @@ var _ = Describe("Test:", func() {
 		})
 
 		It("Checks Node and Pod Status", func() {
+			defer func() {
+				_, err := ParseNodes(kubeConfigFile, true)
+				if err != nil {
+					fmt.Println("Error retrieving nodes: ", err)
+				}
+				_, err = ParsePods(kubeConfigFile, true)
+				if err != nil {
+					fmt.Println("Error retrieving pods: ", err)
+				}
+			}()
+
 			fmt.Printf("\nFetching node status\n")
+			expectedNodeCount := *serverNodes + *workerNodes + 1
 			Eventually(func(g Gomega) {
 				nodes, err := ParseNodes(kubeConfigFile, false)
 				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(len(nodes)).To(Equal(expectedNodeCount), "Number of nodes should match the spec")
 				for _, node := range nodes {
-					g.Expect(node.Status).Should(Equal("Ready"))
+					g.Expect(node.Status).Should(Equal("Ready"), "Nodes should all be in Ready state")
 				}
 			}, "420s", "5s").Should(Succeed())
-			_, _ = ParseNodes(kubeConfigFile, true)
 
-			fmt.Printf("\nFetching Pods status\n")
+			fmt.Printf("\nFetching pod status\n")
 			Eventually(func(g Gomega) {
 				pods, err := ParsePods(kubeConfigFile, false)
 				g.Expect(err).NotTo(HaveOccurred())
@@ -52,10 +68,10 @@ var _ = Describe("Test:", func() {
 						g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
 					} else {
 						g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
+						g.Expect(pod.Restarts).Should(Equal("0"), pod.Name)
 					}
 				}
-			}, "420s", "5s").Should(Succeed())
-			_, _ = ParsePods(kubeConfigFile, true)
+			}, "600s", "5s").Should(Succeed())
 		})
 
 		It("Verifies ClusterIP Service", func() {
@@ -70,15 +86,13 @@ var _ = Describe("Test:", func() {
 			}, "420s", "5s").Should(Succeed())
 
 			clusterip, _ := FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
-			cmd := "curl -L --insecure http://" + clusterip + "/name.html"
-			fmt.Println(cmd)
+			cmd := "curl -sL --insecure http://" + clusterip + "/name.html"
 			nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
 			for _, ip := range nodeExternalIP {
 				Eventually(func(g Gomega) {
-					res, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
+					res, err := RunCmdOnNode(cmd, ip, *sshuser, *access_key)
 					g.Expect(err).NotTo(HaveOccurred())
 					g.Expect(res).Should(ContainSubstring("test-clusterip"))
-					fmt.Println(res)
 				}, "420s", "10s").Should(Succeed())
 			}
 		})
@@ -99,14 +113,11 @@ var _ = Describe("Test:", func() {
 					g.Expect(res).Should(ContainSubstring("test-nodeport"))
 				}, "240s", "5s").Should(Succeed())
 
-				cmd = "curl -L --insecure http://" + ip + ":" + nodeport + "/name.html"
-				fmt.Println(cmd)
+				cmd = "curl -sL --insecure http://" + ip + ":" + nodeport + "/name.html"
 				Eventually(func(g Gomega) {
 					res, err := RunCommand(cmd)
 					g.Expect(err).NotTo(HaveOccurred())
-					fmt.Println(res)
 					g.Expect(res).Should(ContainSubstring("test-nodeport"))
-					fmt.Println(res)
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
@@ -128,13 +139,10 @@ var _ = Describe("Test:", func() {
 				}, "240s", "5s").Should(Succeed())
 
 				Eventually(func(g Gomega) {
-					cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html"
-					fmt.Println(cmd)
+					cmd = "curl -sL --insecure http://" + ip + ":" + port + "/name.html"
 					res, err := RunCommand(cmd)
 					g.Expect(err).NotTo(HaveOccurred())
-					fmt.Println(res)
 					g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
-					fmt.Println(res)
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
@@ -154,14 +162,11 @@ var _ = Describe("Test:", func() {
 			Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned")
 
 			for _, ip := range ingressIps {
-				cmd := "curl  --header host:foo1.bar.com" + " http://" + ip + "/name.html"
-				fmt.Println(cmd)
-
+				cmd := "curl -s --header host:foo1.bar.com" + " http://" + ip + "/name.html"
 				Eventually(func(g Gomega) {
 					res, err := RunCommand(cmd)
 					g.Expect(err).NotTo(HaveOccurred())
 					g.Expect(res).Should(ContainSubstring("test-ingress"))
-					fmt.Println(res)
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
@@ -175,10 +180,6 @@ var _ = Describe("Test:", func() {
 
 			Eventually(func(g Gomega) {
 				count := CountOfStringInSlice("test-daemonset", pods)
-				fmt.Println("POD COUNT")
-				fmt.Println(count)
-				fmt.Println("NODE COUNT")
-				fmt.Println(len(nodes))
 				g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count")
 			}, "420s", "10s").Should(Succeed())
 		})
@@ -191,7 +192,6 @@ var _ = Describe("Test:", func() {
 				cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile
 				res, err := RunCommand(cmd)
 				g.Expect(err).NotTo(HaveOccurred())
-				fmt.Println(res)
 				g.Expect(res).Should(ContainSubstring("local-path-pvc"))
 				g.Expect(res).Should(ContainSubstring("Bound"))
 			}, "420s", "2s").Should(Succeed())
@@ -207,20 +207,18 @@ var _ = Describe("Test:", func() {
 			cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'"
 			_, err = RunCommand(cmd)
 			Expect(err).NotTo(HaveOccurred())
-			fmt.Println("Data stored in pvc: local-path-test")
 
 			cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile
-			res, err := RunCommand(cmd)
+			_, err = RunCommand(cmd)
 			Expect(err).NotTo(HaveOccurred())
-			fmt.Println(res)
 
 			_, err = DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + kubeConfigFile
-				res, _ := RunCommand(cmd)
-				fmt.Println(res)
+				res, err := RunCommand(cmd)
+				Expect(err).NotTo(HaveOccurred())
 				g.Expect(res).Should(ContainSubstring("pod/local-path-provisioner"))
 			}, "420s", "2s").Should(Succeed())
 
@@ -228,16 +226,14 @@ var _ = Describe("Test:", func() {
 				cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
 				res, err := RunCommand(cmd)
 				g.Expect(err).NotTo(HaveOccurred())
-				fmt.Println(res)
 				g.Expect(res).Should(ContainSubstring("volume-test"))
 				g.Expect(res).Should(ContainSubstring("Running"))
 			}, "420s", "2s").Should(Succeed())
 
 			Eventually(func(g Gomega) {
-				cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile
-				res, err = RunCommand(cmd)
+				cmd = "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test"
+				res, err := RunCommand(cmd)
 				g.Expect(err).NotTo(HaveOccurred())
-				fmt.Println("Data after re-creation", res)
 				g.Expect(res).Should(ContainSubstring("local-path-test"))
 			}, "180s", "2s").Should(Succeed())
 		})
@@ -249,7 +245,6 @@ var _ = Describe("Test:", func() {
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile
 				res, _ := RunCommand(cmd)
-				fmt.Println(res)
 				g.Expect(res).Should(ContainSubstring("dnsutils"))
 				g.Expect(res).Should(ContainSubstring("Running"))
 			}, "420s", "2s").Should(Succeed())
@@ -257,53 +252,18 @@ var _ = Describe("Test:", func() {
 			Eventually(func(g Gomega) {
 				cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default"
 				res, _ := RunCommand(cmd)
-				fmt.Println(res)
 				g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local"))
-				fmt.Println(res)
 			}, "420s", "2s").Should(Succeed())
 		})
-
-		It("Validate Rebooting nodes", func() {
-			nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
-			for _, ip := range nodeExternalIP {
-				fmt.Println("\nRebooting node: ", ip)
-				cmd := "ssh -i " + *sshkey + " -o \"StrictHostKeyChecking no\" " + *sshuser + "@" + ip + " sudo reboot"
-				_, _ = RunCommand(cmd)
-				time.Sleep(3 * time.Minute)
-				fmt.Println("\nNode and Pod Status after rebooting node: ", ip)
-
-				Eventually(func(g Gomega) {
-					nodes, err := ParseNodes(kubeConfigFile, false)
-					g.Expect(err).NotTo(HaveOccurred())
-					for _, node := range nodes {
-						g.Expect(node.Status).Should(Equal("Ready"))
-					}
-				}, "420s", "5s").Should(Succeed())
-				_, _ = ParseNodes(kubeConfigFile, true)
-
-				Eventually(func(g Gomega) {
-					pods, err := ParsePods(kubeConfigFile, false)
-					g.Expect(err).NotTo(HaveOccurred())
-					for _, pod := range pods {
-						if strings.Contains(pod.Name, "helm-install") {
-							g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
-						} else {
-							g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
-						}
-					}
-				}, "420s", "5s").Should(Succeed())
-				_, _ = ParsePods(kubeConfigFile, true)
-			}
-		})
 	})
 })
 
 var _ = AfterEach(func() {
-	failed = failed || CurrentGinkgoTestDescription().Failed
+	failed = failed || CurrentSpecReport().Failed()
 })
 
 var _ = BeforeEach(func() {
-	failed = failed || CurrentGinkgoTestDescription().Failed
+	failed = failed || CurrentSpecReport().Failed()
 	if *destroy {
 		fmt.Printf("\nCluster is being Deleted\n")
 		Skip("Cluster is being Deleted")
@@ -313,10 +273,11 @@ var _ = BeforeEach(func() {
 var _ = AfterSuite(func() {
 	if failed {
 		fmt.Println("FAILED!")
+	} else if *destroy {
+		status, err := BuildCluster(*nodeOs, *awsAmi, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(status).To(Equal("cluster destroyed"))
 	} else {
-		kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true, *arch)
-		if err != nil {
-			fmt.Println("Error Destroying Cluster", err)
-		}
+		fmt.Println("PASSED!")
 	}
 })

tests/terraform/jenkinsfile

@@ -78,8 +78,8 @@ pipeline {
                         /usr/bin/docker cp "${WORKSPACE}/tests/terraform/$AWS_SSH_KEY_NAME" "${RESOURCE_NAME}_${BUILD_NUMBER}":/config
 
                         /usr/bin/docker exec ${RESOURCE_NAME}_${BUILD_NUMBER} /usr/local/go/bin/go test -v tests/terraform/createcluster.go \
-                        tests/terraform/createcluster_test.go tests/terraform/testutils.go -v \
-                        -timeout=2h -node_os=${NODE_OS} \
+                        tests/terraform/createcluster_test.go tests/terraform/testutils.go \
+                        -timeout=1h -node_os=${NODE_OS} \
                         -cluster_type=${CLUSTER_TYPE} -external_db=${EXTERNAL_DB} -resource_name=${RESOURCE_NAME} \
                         -sshuser=${AWS_USER} -sshkey="/config/${AWS_SSH_KEY_NAME}" -destroy=false -arch=${ARCH}
 

tests/terraform/modules/k3scluster/config/local.tfvars

@@ -0,0 +1,28 @@
+region = "us-east-2"
+qa_space = ""
+create_lb = false
+
+external_db_version = "5.7"
+instance_class = "db.t2.micro"
+db_group_name = "mysql5.7"
+engine_mode = "provisioned"
+db_username = ""
+db_password = ""
+
+username = ""
+password = ""
+
+ec2_instance_class = "t3a.medium"
+vpc_id = ""
+subnets = ""
+availability_zone = "us-east-2a"
+sg_id = ""
+
+no_of_server_nodes = 2
+no_of_worker_nodes = 1
+server_flags = "token: test"
+worker_flags = "token: test"
+
+k3s_version = "v1.23.8+k3s2"
+install_mode = "INSTALL_K3S_VERSION"
+environment = "local"

tests/terraform/modules/k3scluster/master/instances_server.tf

@@ -90,9 +90,6 @@ resource "aws_instance" "master" {
   provisioner "local-exec" {
     command = "sed s/127.0.0.1/\"${var.create_lb ? aws_route53_record.aws_route53[0].fqdn : aws_instance.master.public_ip}\"/g /tmp/${var.resource_name}_config >/tmp/${var.resource_name}_kubeconfig"
   }
-  provisioner "local-exec" {
-    command = "sed s/127.0.0.1/\"${var.create_lb ? aws_route53_record.aws_route53[0].fqdn : aws_instance.master.public_ip}\"/g /tmp/${var.resource_name}_config >/config/${var.resource_name}_kubeconfig"
-  }
 }
 
 data "template_file" "test" {

tests/terraform/scripts/build.sh

@@ -1 +0,0 @@
-/usr/local/bin/docker build -f Dockerfile.build -t k3s_create_cluster .

tests/terraform/testutils.go

@@ -1,4 +1,4 @@
-package e2e
+package terraform
 
 import (
 	"bytes"

tests/terraform/upgradecluster_test.go

@@ -1,4 +1,4 @@
-package e2e
+package terraform
 
 import (
 	"flag"
@@ -12,7 +12,7 @@ import (
 
 var upgradeVersion = flag.String("upgrade_version", "", "a string")
 
-func Test_E2EClusterUpgradeValidation(t *testing.T) {
+func Test_TFClusterUpgradeValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
 	RunSpecs(t, "Upgrade Cluster Test Suite")
@@ -21,8 +21,9 @@ func Test_E2EClusterUpgradeValidation(t *testing.T) {
 var _ = Describe("Test:", func() {
 	Context("Build Cluster:", func() {
 		It("Starts up with no issues", func() {
-			kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
+			status, err := BuildCluster(*nodeOs, *awsAmi, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
 			Expect(err).NotTo(HaveOccurred())
+			Expect(status).To(Equal("cluster created"))
 			defer GinkgoRecover()
 			fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "BACKEND", *clusterType, *externalDb)
 			fmt.Printf("\nIPs:\n")
@@ -503,13 +504,14 @@ var _ = BeforeEach(func() {
 	}
 
 })
-var _ = AfterSuite(func() {
-	if failed {
-		fmt.Println("FAILED!")
-	} else {
-		kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
-		if err != nil {
-			fmt.Println("Error Destroying Cluster", err)
-		}
-	}
-})
+
+// var _ = AfterSuite(func() {
+// 	if failed {
+// 		fmt.Println("FAILED!")
+// 	} else {
+// 		kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *awsAmi, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
+// 		if err != nil {
+// 			fmt.Println("Error Destroying Cluster", err)
+// 		}
+// 	}
+// })