Redux: Enable K3s integration test to run on existing cluster (#3905)

* Made it possible to run int tests on existing cluster

Signed-off-by: dereknola <derek.nola@suse.com>

pkg/etcd/etcd_int_test.go

@@ -12,13 +12,21 @@ import (
 )
 
 var server *testutil.K3sServer
+var serverArgs = []string{"--cluster-init"}
 var _ = BeforeSuite(func() {
-	var err error
+	if !testutil.IsExistingServer() {
-	server, err = testutil.K3sStartServer("--cluster-init")
+		var err error
-	Expect(err).ToNot(HaveOccurred())
+		server, err = testutil.K3sStartServer(serverArgs...)
+		Expect(err).ToNot(HaveOccurred())
+	}
 })
 
 var _ = Describe("etcd snapshots", func() {
+	BeforeEach(func() {
+		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) {
+			Skip("Test needs k3s server with: " + strings.Join(serverArgs, " "))
+		}
+	})
 	When("a new etcd is created", func() {
 		It("starts up with no problems", func() {
 			Eventually(func() (string, error) {
@@ -44,9 +52,6 @@ var _ = Describe("etcd snapshots", func() {
 		})
 	})
 	When("saving a custom name", func() {
-		It("starts with no snapshots", func() {
-			Expect(testutil.K3sCmd("etcd-snapshot", "ls")).To(BeEmpty())
-		})
 		It("saves an etcd snapshot with a custom name", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")).
 				To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/k3s/server/db/snapshots/ALIVEBEEF"))
@@ -62,9 +67,6 @@ var _ = Describe("etcd snapshots", func() {
 		})
 	})
 	When("using etcd snapshot prune", func() {
-		It("starts with no snapshots", func() {
-			Expect(testutil.K3sCmd("etcd-snapshot", "ls")).To(BeEmpty())
-		})
 		It("saves 3 different snapshots", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
 				To(ContainSubstring("Saving current etcd snapshot set to k3s-etcd-snapshots"))
@@ -79,10 +81,9 @@ var _ = Describe("etcd snapshots", func() {
 		It("lists all 3 snapshots", func() {
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
-			sepLines := strings.FieldsFunc(lsResult, func(c rune) bool {
+			reg, err := regexp.Compile(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)
-				return c == '\n'
+			Expect(err).ToNot(HaveOccurred())
-			})
+			sepLines := reg.FindAllString(lsResult, -1)
-			Expect(lsResult).To(MatchRegexp(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`))
 			Expect(sepLines).To(HaveLen(3))
 		})
 		It("prunes snapshots down to 2", func() {
@@ -90,10 +91,9 @@ var _ = Describe("etcd snapshots", func() {
 				To(BeEmpty())
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
-			sepLines := strings.FieldsFunc(lsResult, func(c rune) bool {
+			reg, err := regexp.Compile(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)
-				return c == '\n'
+			Expect(err).ToNot(HaveOccurred())
-			})
+			sepLines := reg.FindAllString(lsResult, -1)
-			Expect(lsResult).To(MatchRegexp(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`))
 			Expect(sepLines).To(HaveLen(2))
 		})
 		It("cleans up remaining snapshots", func() {
@@ -110,7 +110,9 @@ var _ = Describe("etcd snapshots", func() {
 })
 
 var _ = AfterSuite(func() {
-	Expect(testutil.K3sKillServer(server)).To(Succeed())
+	if !testutil.IsExistingServer() {
+		Expect(testutil.K3sKillServer(server)).To(Succeed())
+	}
 })
 
 func Test_IntegrationEtcd(t *testing.T) {

tests/TESTING.md

@@ -71,10 +71,16 @@ See the [local storage test](https://github.com/k3s-io/k3s/blob/master/tests/int
 
 ### Running
 
+Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs.
 ```bash
 go test ./pkg/... ./tests/... -run Integration
 ```
 
+Integration tests can also be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly.
+```
+go test -ldflags "-X 'github.com/rancher/k3s/tests/util.existingServer=True'" ./pkg/... ./tests/... -run Integration
+```
+
 ___
 
 ## End-to-End (E2E) Tests

tests/integration/localstorage_int_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -12,13 +13,21 @@ import (
 )
 
 var server *testutil.K3sServer
+var serverArgs = []string{"--cluster-init"}
 var _ = BeforeSuite(func() {
-	var err error
+	if !testutil.IsExistingServer() {
-	server, err = testutil.K3sStartServer("--cluster-init")
+		var err error
-	Expect(err).ToNot(HaveOccurred())
+		server, err = testutil.K3sStartServer(serverArgs...)
+		Expect(err).ToNot(HaveOccurred())
+	}
 })
 
 var _ = Describe("local storage", func() {
+	BeforeEach(func() {
+		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) {
+			Skip("Test needs k3s server with: " + strings.Join(serverArgs, " "))
+		}
+	})
 	When("a new local storage is created", func() {
 		It("starts up with no problems", func() {
 			Eventually(func() (string, error) {
@@ -66,11 +75,12 @@ var _ = Describe("local storage", func() {
 })
 
 var _ = AfterSuite(func() {
-	Expect(testutil.K3sKillServer(server)).To(Succeed())
+	if !testutil.IsExistingServer() {
+		Expect(testutil.K3sKillServer(server)).To(Succeed())
+	}
 })
 
 func Test_IntegrationLocalStorage(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Local Storage Suite")
-
 }

tests/util/cmd.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"bufio"
+	"encoding/json"
 	"os"
 	"os/exec"
 	"os/user"
@@ -11,7 +12,18 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// Compile-time variable
+var existingServer = "False"
+
 func findK3sExecutable() string {
+	// if running on an existing cluster, it maybe installed via k3s.service
+	// or run manually from dist/artifacts/k3s
+	if IsExistingServer() {
+		k3sBin, err := exec.LookPath("k3s")
+		if err == nil {
+			return k3sBin
+		}
+	}
 	k3sBin := "dist/artifacts/k3s"
 	for {
 		_, err := os.Stat(k3sBin)
@@ -33,6 +45,10 @@ func IsRoot() bool {
 	return currentUser.Uid == "0"
 }
 
+func IsExistingServer() bool {
+	return existingServer == "True"
+}
+
 // K3sCmd launches the provided K3s command via exec. Command blocks until finished.
 // Command output from both Stderr and Stdout is provided via string.
 //   cmdEx1, err := K3sCmd("etcd-snapshot", "ls")
@@ -52,6 +68,41 @@ func K3sCmd(cmdName string, cmdArgs ...string) (string, error) {
 	return string(byteOut), err
 }
 
+func contains(source []string, target string) bool {
+	for _, s := range source {
+		if s == target {
+			return true
+		}
+	}
+	return false
+}
+
+// ServerArgsPresent checks if the given arguments are found in the running k3s server
+func ServerArgsPresent(neededArgs []string) bool {
+	currentArgs := K3sServerArgs()
+	for _, arg := range neededArgs {
+		if !contains(currentArgs, arg) {
+			return false
+		}
+	}
+	return true
+}
+
+// K3sServerArgs returns the list of arguments that the k3s server launched with
+func K3sServerArgs() []string {
+	results, err := K3sCmd("kubectl", "get", "nodes", "-o", `jsonpath='{.items[0].metadata.annotations.k3s\.io/node-args}'`)
+	if err != nil {
+		return nil
+	}
+	res := strings.ReplaceAll(results, "'", "")
+	var args []string
+	if err := json.Unmarshal([]byte(res), &args); err != nil {
+		logrus.Error(err)
+		return nil
+	}
+	return args
+}
+
 func FindStringInCmdAsync(scanner *bufio.Scanner, target string) bool {
 	for scanner.Scan() {
 		if strings.Contains(scanner.Text(), target) {