From 75f8cfb6ea224488a91cf16f71c55085a025de93 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 15 Aug 2022 15:00:22 -0700 Subject: [PATCH] E2E: Local cluster testing (#5977) * Cleanup validatecluster printout * remove deprecated kubectl exec format * Implement CreateLocalCluster function * Update testing documentation Signed-off-by: Derek Nola --- tests/e2e/README.md | 26 ++++++++ tests/e2e/docker/Vagrantfile | 1 + tests/e2e/dualstack/Vagrantfile | 7 +- tests/e2e/scripts/ipv6.sh | 6 ++ tests/e2e/testutils.go | 66 +++++++++++++++++-- .../e2e/upgradecluster/upgradecluster_test.go | 4 +- .../validatecluster/validatecluster_test.go | 41 +++++------- 7 files changed, 114 insertions(+), 37 deletions(-) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index b55679e14d..cb73bc5b5c 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -20,6 +20,32 @@ A E2E test consists of two parts: See the [validate cluster test](../tests/e2e/validatecluster/validatecluster_test.go) as an example. + +## Setup + +To run the E2E tests, you must first install the following: +- Vagrant +- Libvirt +- Vagrant plugins + +### Vagrant + +Download the latest version (currently 2.2.19) of Vagrant [*from the website*](https://www.vagrantup.com/downloads). Do not use built-in packages, they often old or do not include the required ruby library extensions necessary to get certain plugins working. + +### Libvirt +Follow the OS specific guides to install libvirt/qemu on your host: + - [openSUSE](https://documentation.suse.com/sles/15-SP1/html/SLES-all/cha-vt-installation.html) + - [ubuntu](https://ubuntu.com/server/docs/virtualization-libvirt) + - [debian](https://wiki.debian.org/KVM#Installation) + - [fedora](https://developer.fedoraproject.org/tools/virtualization/installing-libvirt-and-virt-install-on-fedora-linux.html) + +### Vagrant plugins +Install the necessary vagrant plugins with the following command: + +```bash +vagrant plugin install vagrant-libvirt vagrant-scp vagrant-k3s vagrant-reload +``` + ## Running Generally, E2E tests are run as a nightly Jenkins job for QA. They can still be run locally but additional setup may be required. By default, all E2E tests are designed with `libvirt` as the underlying VM provider. Instructions for installing libvirt and its associated vagrant plugin, `vagrant-libvirt` can be found [here.](https://github.com/vagrant-libvirt/vagrant-libvirt#installation) `VirtualBox` is also supported as a backup VM provider. diff --git a/tests/e2e/docker/Vagrantfile b/tests/e2e/docker/Vagrantfile index 74d04b4c33..81be33ac45 100644 --- a/tests/e2e/docker/Vagrantfile +++ b/tests/e2e/docker/Vagrantfile @@ -36,6 +36,7 @@ def provision(vm, role, role_num, node_num) vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.args = %W[agent --server https://#{NETWORK_PREFIX}.100:6443 --flannel-iface=eth1 --docker] k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}] + k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 end end diff --git a/tests/e2e/dualstack/Vagrantfile b/tests/e2e/dualstack/Vagrantfile index 84342ef7ae..46c9b7e8ac 100644 --- a/tests/e2e/dualstack/Vagrantfile +++ b/tests/e2e/dualstack/Vagrantfile @@ -26,12 +26,13 @@ def provision(vm, roles, role_num, node_num) :libvirt__ipv6_address => "#{NETWORK6_PREFIX}::1", :libvirt__ipv6_prefix => "64" - vagrant_defaults = '../vagrantdefaults.rb' - load vagrant_defaults if File.exists?(vagrant_defaults) + scripts_location = Dir.exists?("./scripts") ? "./scripts" : "../scripts" + vagrant_defaults = File.exists?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb" + load vagrant_defaults defaultOSConfigure(vm) - vm.provision "IPv6 Setup", type: "shell", path: "../scripts/ipv6.sh", args: [node_ip4, node_ip6, vm.box] + vm.provision "IPv6 Setup", type: "shell", path: scripts_location +"/ipv6.sh", args: [node_ip4, node_ip6, vm.box.to_s] install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) vm.provision "Ping Check", type: "shell", inline: "ping -c 2 k3s.io" diff --git a/tests/e2e/scripts/ipv6.sh b/tests/e2e/scripts/ipv6.sh index af7f823a93..5dbf6b4c53 100644 --- a/tests/e2e/scripts/ipv6.sh +++ b/tests/e2e/scripts/ipv6.sh @@ -5,11 +5,17 @@ os=$3 sysctl -w net.ipv6.conf.all.disable_ipv6=0 sysctl -w net.ipv6.conf.eth1.accept_dad=0 +sysctl -w net.ipv6.conf.eth1.accept_ra=0 +sysctl -w net.ipv6.conf.eth1.forwarding=0 if [ -z "${os##*ubuntu*}" ]; then netplan set ethernets.eth1.accept-ra=false netplan set ethernets.eth1.addresses=["$ip4_addr"/24,"$ip6_addr"/64] netplan apply +elif [ -z "${os##*alpine*}" ]; then + iplink set eth1 down + iplink set eth1 up + ip -6 addr add "$ip6_addr"/64 dev eth1 else ip -6 addr add "$ip6_addr"/64 dev eth1 fi diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index b279632645..2980ecc343 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -38,7 +38,8 @@ func CountOfStringInSlice(str string, pods []Pod) int { return count } -func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []string, error) { +// genNodeEnvs generates the node and testing environment variables for vagrant up +func genNodeEnvs(nodeOS string, serverCount, agentCount int) ([]string, []string, string) { serverNodeNames := make([]string, serverCount) for i := 0; i < serverCount; i++ { serverNodeNames[i] = "server-" + strconv.Itoa(i) @@ -47,12 +48,22 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri for i := 0; i < agentCount; i++ { agentNodeNames[i] = "agent-" + strconv.Itoa(i) } - nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(agentNodeNames, " ") + nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(agentNodeNames, " ") nodeRoles = strings.TrimSpace(nodeRoles) + nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+agentCount) nodeBoxes = strings.TrimSpace(nodeBoxes) + nodeEnvs := fmt.Sprintf(`E2E_NODE_ROLES="%s" E2E_NODE_BOXES="%s"`, nodeRoles, nodeBoxes) + + return serverNodeNames, agentNodeNames, nodeEnvs +} + +func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []string, error) { + + serverNodeNames, agentNodeNames, nodeEnvs := genNodeEnvs(nodeOS, serverCount, agentCount) + var testOptions string for _, env := range os.Environ() { if strings.HasPrefix(env, "E2E_") { @@ -60,15 +71,54 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri } } - cmd := fmt.Sprintf(`E2E_NODE_ROLES="%s" E2E_NODE_BOXES="%s" %s vagrant up &> vagrant.log`, nodeRoles, nodeBoxes, testOptions) + cmd := fmt.Sprintf(`%s %s vagrant up &> vagrant.log`, nodeEnvs, testOptions) fmt.Println(cmd) if _, err := RunCommand(cmd); err != nil { - fmt.Println("Error Creating Cluster", err) - return nil, nil, err + return nil, nil, fmt.Errorf("failed creating cluster: %s: %v", cmd, err) } return serverNodeNames, agentNodeNames, nil } +// CreateLocalCluster creates a cluster using the locally built k3s binary. The vagrant-scp plugin must be installed for +// this function to work. The binary is deployed as an airgapped install of k3s on the VMs. +// This is intended only for local testing puposes when writing a new E2E test. +func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, []string, error) { + + serverNodeNames, agentNodeNames, nodeEnvs := genNodeEnvs(nodeOS, serverCount, agentCount) + + var testOptions string + for _, env := range os.Environ() { + if strings.HasPrefix(env, "E2E_") { + testOptions += " " + env + } + } + testOptions += " E2E_RELEASE_VERSION=skip" + + cmd := fmt.Sprintf(`%s vagrant up --no-provision &> vagrant.log`, nodeEnvs) + if _, err := RunCommand(cmd); err != nil { + return nil, nil, fmt.Errorf("failed creating nodes: %s: %v", cmd, err) + } + + nodeRoles := append(serverNodeNames, agentNodeNames...) + + for _, node := range nodeRoles { + cmd = fmt.Sprintf(`vagrant scp ../../../dist/artifacts/k3s %s:/tmp/`, node) + if _, err := RunCommand(cmd); err != nil { + return nil, nil, fmt.Errorf("failed to scp k3s binary to %s: %v", node, err) + } + if _, err := RunCmdOnNode("sudo mv /tmp/k3s /usr/local/bin/", node); err != nil { + return nil, nil, err + } + } + + cmd = fmt.Sprintf(`%s %s vagrant provision &>> vagrant.log`, nodeEnvs, testOptions) + if _, err := RunCommand(cmd); err != nil { + return nil, nil, fmt.Errorf("failed creating cluster: %s: %v", cmd, err) + } + + return serverNodeNames, agentNodeNames, nil +} + func DeployWorkload(workload, kubeconfig string, hardened bool) (string, error) { resourceDir := "../amd64_resource_files" if hardened { @@ -240,7 +290,11 @@ func RestartCluster(nodeNames []string) error { // RunCmdOnNode executes a command from within the given node func RunCmdOnNode(cmd string, nodename string) (string, error) { runcmd := "vagrant ssh -c \"" + cmd + "\" " + nodename - return RunCommand(runcmd) + out, err := RunCommand(runcmd) + if err != nil { + return out, fmt.Errorf("failed to run command %s on node %s: %v", cmd, nodename, err) + } + return out, nil } // RunCommand executes a command on the host diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index 4603f3d780..033d3a0517 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -234,7 +234,7 @@ var _ = Describe("Verify Upgrade", func() { // Check data after re-creation Eventually(func() (string, error) { - cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile + cmd := "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test" return e2e.RunCommand(cmd) }, "180s", "2s").Should(ContainSubstring("local-path-test")) }) @@ -364,7 +364,7 @@ var _ = Describe("Verify Upgrade", func() { It("After upgrade verify Local Path Provisioner storage ", func() { Eventually(func() (string, error) { - cmd := "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile + cmd := "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test" return e2e.RunCommand(cmd) }, "180s", "2s").Should(ContainSubstring("local-path-test")) }) diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index a65a580625..65afc1d0c7 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -92,7 +92,6 @@ var _ = Describe("Verify Create", func() { Eventually(func(g Gomega) { res, err := e2e.RunCmdOnNode(cmd, nodeName) g.Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) Expect(res).Should(ContainSubstring("test-clusterip")) }, "120s", "10s").Should(Succeed()) } @@ -119,8 +118,7 @@ var _ = Describe("Verify Create", func() { fmt.Println(cmd) Eventually(func(g Gomega) { res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("test-nodeport")) }, "240s", "5s").Should(Succeed()) } @@ -140,16 +138,14 @@ var _ = Describe("Verify Create", func() { Eventually(func(g Gomega) { cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("test-loadbalancer")) }, "240s", "5s").Should(Succeed()) Eventually(func(g Gomega) { cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html" - fmt.Println(cmd) res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("test-loadbalancer")) }, "240s", "5s").Should(Succeed()) } @@ -166,8 +162,7 @@ var _ = Describe("Verify Create", func() { Eventually(func(g Gomega) { res, err := e2e.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("test-ingress")) }, "240s", "5s").Should(Succeed()) } @@ -196,16 +191,15 @@ var _ = Describe("Verify Create", func() { Eventually(func(g Gomega) { cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile - res, _ := e2e.RunCommand(cmd) - fmt.Println(res) + res, err := e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("dnsutils")) }, "420s", "2s").Should(Succeed()) Eventually(func(g Gomega) { cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default" - fmt.Println(cmd) - res, _ := e2e.RunCommand(cmd) - fmt.Println(res) + res, err := e2e.RunCommand(cmd) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) }, "420s", "2s").Should(Succeed()) }) @@ -217,8 +211,7 @@ var _ = Describe("Verify Create", func() { Eventually(func(g Gomega) { cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile res, err := e2e.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("local-path-pvc")) g.Expect(res).Should(ContainSubstring("Bound")) }, "420s", "2s").Should(Succeed()) @@ -226,8 +219,7 @@ var _ = Describe("Verify Create", func() { Eventually(func(g Gomega) { cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) g.Expect(res).Should(ContainSubstring("volume-test")) g.Expect(res).Should(ContainSubstring("Running")) }, "420s", "2s").Should(Succeed()) @@ -235,12 +227,10 @@ var _ = Describe("Verify Create", func() { cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'" _, err = e2e.RunCommand(cmd) Expect(err).NotTo(HaveOccurred()) - fmt.Println("Data stored in pvc: local-path-test") cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) + Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) _, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") @@ -248,23 +238,22 @@ var _ = Describe("Verify Create", func() { Eventually(func(g Gomega) { cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + kubeConfigFile res, _ := e2e.RunCommand(cmd) - fmt.Println(res) g.Expect(res).Should(ContainSubstring("local-path-provisioner")) }, "420s", "2s").Should(Succeed()) Eventually(func(g Gomega) { cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile res, err := e2e.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) + g.Expect(res).Should(ContainSubstring("volume-test")) g.Expect(res).Should(ContainSubstring("Running")) }, "420s", "2s").Should(Succeed()) Eventually(func(g Gomega) { - cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile + cmd := "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test" res, err = e2e.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) + g.Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd+" result: "+res) fmt.Println("Data after re-creation", res) g.Expect(res).Should(ContainSubstring("local-path-test")) }, "180s", "2s").Should(Succeed())