Add new `prefer-bundled-bin` experimental flag (#6420)

* initial prefer-bundled-bin ci change
* Add startup testlet
* Convert parsing to pflag library
* Fix code validation
* go mod tidy

Signed-off-by: Derek Nola <derek.nola@suse.com>

cmd/k3s/main.go

@@ -3,9 +3,11 @@ package main
 import (
 	"bytes"
 	"context"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
 
@@ -20,6 +22,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
 	"github.com/urfave/cli"
 )
 
@@ -27,7 +30,7 @@ var criDefaultConfigPath = "/etc/crictl.yaml"
 
 // main entrypoint for the k3s multicall binary
 func main() {
-	dataDir := findDataDir()
+	dataDir := findDataDir(os.Args)
 
 	// Handle direct invocation via symlink alias (multicall binary behavior)
 	if runCLIs(dataDir) {
@@ -79,19 +82,17 @@ func main() {
 // findDataDir reads data-dir settings from the CLI args and config file.
 // If not found, the default will be used, which varies depending on whether
 // k3s is being run as root or not.
-func findDataDir() string {
-	for i, arg := range os.Args {
-		for _, flagName := range []string{"--data-dir", "-d"} {
-			if flagName == arg {
-				if len(os.Args) > i+1 {
-					return os.Args[i+1]
-				}
-			} else if strings.HasPrefix(arg, flagName+"=") {
-				return arg[len(flagName)+1:]
-			}
-		}
+func findDataDir(args []string) string {
+	var dataDir string
+	fs := pflag.NewFlagSet("data-dir-set", pflag.ContinueOnError)
+	fs.ParseErrorsWhitelist.UnknownFlags = true
+	fs.SetOutput(io.Discard)
+	fs.StringVarP(&dataDir, "data-dir", "d", "", "Data directory")
+	fs.Parse(args)
+	if dataDir != "" {
+		return dataDir
 	}
-	dataDir := configfilearg.MustFindString(os.Args, "data-dir")
+	dataDir = configfilearg.MustFindString(args, "data-dir")
 	if d, err := datadir.Resolve(dataDir); err == nil {
 		dataDir = d
 	} else {
@@ -100,6 +101,24 @@ func findDataDir() string {
 	return dataDir
 }
 
+// findPreferBundledBin searches for prefer-bundled-bin from the config file, then CLI args.
+// we use pflag to process the args because we not yet parsed flags bound to the cli.Context
+func findPreferBundledBin(args []string) bool {
+	var preferBundledBin bool
+	fs := pflag.NewFlagSet("prefer-set", pflag.ContinueOnError)
+	fs.ParseErrorsWhitelist.UnknownFlags = true
+	fs.SetOutput(io.Discard)
+	fs.BoolVar(&preferBundledBin, "prefer-bundled-bin", false, "Prefer bundled binaries")
+
+	preferRes := configfilearg.MustFindString(args, "prefer-bundled-bin")
+	if preferRes != "" {
+		preferBundledBin, _ = strconv.ParseBool(preferRes)
+	}
+
+	fs.Parse(args)
+	return preferBundledBin
+}
+
 // runCLIs handles the case where the binary is being executed as a symlink alias,
 // /usr/local/bin/crictl for example. If the executable name is one of the external
 // binaries, it calls it directly and returns true. If it's not an external binary,
@@ -158,7 +177,13 @@ func stageAndRun(dataDir, cmd string, args []string) error {
 	}
 	logrus.Debugf("Asset dir %s", dir)
 
-	if err := os.Setenv("PATH", filepath.Join(dir, "bin")+":"+os.Getenv("PATH")+":"+filepath.Join(dir, "bin/aux")); err != nil {
+	var pathEnv string
+	if findPreferBundledBin(args) {
+		pathEnv = filepath.Join(dir, "bin") + ":" + filepath.Join(dir, "bin/aux") + ":" + os.Getenv("PATH")
+	} else {
+		pathEnv = filepath.Join(dir, "bin") + ":" + os.Getenv("PATH") + ":" + filepath.Join(dir, "bin/aux")
+	}
+	if err := os.Setenv("PATH", pathEnv); err != nil {
 		return err
 	}
 	if err := os.Setenv(version.ProgramUpper+"_DATA_DIR", dir); err != nil {

cmd/k3s/main_test.go

@@ -0,0 +1,59 @@
+package main
+
+import "testing"
+
+func Test_UnitFindPreferBundledBin(t *testing.T) {
+	tests := []struct {
+		name string
+		args []string
+		want bool
+	}{
+		{
+			name: "Single argument",
+			args: []string{"--prefer-bundled-bin"},
+			want: true,
+		},
+		{
+			name: "no argument",
+			args: []string{""},
+			want: false,
+		},
+		{
+			name: "Argument with equal true",
+			args: []string{"--prefer-bundled-bin=true"},
+			want: true,
+		},
+		{
+			name: "Argument with equal false",
+			args: []string{"--prefer-bundled-bin=false"},
+			want: false,
+		},
+		{
+			name: "Argument with equal 1",
+			args: []string{"--prefer-bundled-bin=1"},
+			want: true,
+		},
+		{
+			name: "Argument with equal 0",
+			args: []string{"--prefer-bundled-bin=0"},
+			want: false,
+		},
+		{
+			name: "Multiple arguments",
+			args: []string{"--abcd", "--prefer-bundled-bin", "--efgh"},
+			want: true,
+		},
+		{
+			name: "Repeated arguments",
+			args: []string{"--abcd", "--prefer-bundled-bin=false", "--prefer-bundled-bin"},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := findPreferBundledBin(tt.args); got != tt.want {
+				t.Errorf("findPreferBundledBin() = %+v\nWant = %+v", got, tt.want)
+			}
+		})
+	}
+}

go.mod

@@ -109,6 +109,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rootless-containers/rootlesskit v1.0.1
 	github.com/sirupsen/logrus v1.9.0
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.0
 	github.com/urfave/cli v1.22.9
 	github.com/vishvananda/netlink v1.2.1-beta.2
@@ -324,7 +325,6 @@ require (
 	github.com/shengdoushi/base58 v1.0.0 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/spf13/cobra v1.4.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/stretchr/objx v0.4.0 // indirect

pkg/cli/cmds/server.go

@@ -508,6 +508,10 @@ var ServerFlags = []cli.Flag{
 		Usage:       "(experimental) Run rootless",
 		Destination: &ServerConfig.Rootless,
 	},
+	cli.BoolFlag{
+		Name:  "prefer-bundled-bin",
+		Usage: "(experimental) Prefer bundled userspace binaries over host binaries",
+	},
 	cli.BoolFlag{
 		Name:        "secrets-encryption",
 		Usage:       "(experimental) Enable Secret encryption at rest",

scripts/validate

@@ -2,7 +2,6 @@
 set -e
 
 cd $(dirname $0)/..
-. ./scripts/version.sh
 
 echo Running: go mod tidy
 go mod tidy
@@ -16,6 +15,15 @@ if [ -n "$SKIP_VALIDATE" ]; then
 fi
 echo Running validation
 
+. ./scripts/version.sh
+
+if [ -n "$DIRTY" ]; then
+    echo Source dir is dirty
+    git status --porcelain --untracked-files=no
+    git diff
+    exit 1
+fi
+
 echo Running: go version
 DEPENDENCIES_URL="https://raw.githubusercontent.com/kubernetes/kubernetes/${VERSION_K8S}/build/dependencies.yaml"
 GOLANG_VERSION=$(curl -sL "${DEPENDENCIES_URL}" | yq e '.dependencies[] | select(.name == "golang: upstream version").version' -)
@@ -31,13 +39,6 @@ if [ ! -e build/data ];then
     mkdir -p build/data
 fi
 
-if [ -n "$DIRTY" ]; then
-    echo Source dir is dirty
-    git status --porcelain --untracked-files=no
-    git diff
-    exit 1
-fi
-
 if ! command -v golangci-lint; then
     echo Skipping validation: no golangci-lint available
     exit

tests/integration/startup/startup_int_test.go

@@ -185,6 +185,27 @@ var _ = Describe("startup tests", Ordered, func() {
 		})
 		It("dies cleanly", func() {
 			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
+		})
+	})
+	When("a server with prefer-bundled-bin option", func() {
+		It("is created with prefer-bundled-bin flag", func() {
+			var err error
+			startupServerArgs = []string{"--prefer-bundled-bin"}
+			startupServer, err = testutil.K3sStartServer(startupServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("has the default pods deployed", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "120s", "5s").Should(Succeed())
+			nodes, err := testutil.ParseNodes()
+			Expect(err).NotTo(HaveOccurred())
+			Expect(nodes).To(HaveLen(1))
+		})
+		It("dies cleanly", func() {
+			Expect(testutil.K3sKillServer(startupServer)).To(Succeed())
+			Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
 		})
 	})
 })