From 03532f7c0bbd8b83d1a8f300b37b843dbd1c7753 Mon Sep 17 00:00:00 2001 From: Vitor Savian Date: Tue, 21 Nov 2023 20:38:56 +0000 Subject: [PATCH] Added runtime classes for crun/wasm/nvidia Signed-off-by: Vitor Savian Added default runtime flag Signed-off-by: Vitor Savian --- manifests/runtimes.yaml | 59 +++ pkg/agent/containerd/config_linux.go | 9 +- pkg/agent/containerd/nvidia.go | 37 -- pkg/agent/containerd/nvidia_test.go | 267 ----------- pkg/agent/containerd/runtimes.go | 121 ++++- pkg/agent/containerd/runtimes_test.go | 584 ++++++++++++++++++++++++- pkg/agent/containerd/runwasi.go | 60 --- pkg/agent/containerd/runwasi_test.go | 199 --------- pkg/agent/run_linux.go | 5 + pkg/agent/templates/templates_linux.go | 1 + pkg/cli/cmds/agent.go | 7 + pkg/cli/cmds/stage.go | 2 +- pkg/daemons/config/types.go | 1 + pkg/deploy/zz_generated_bindata.go | 23 + 14 files changed, 799 insertions(+), 576 deletions(-) create mode 100644 manifests/runtimes.yaml delete mode 100644 pkg/agent/containerd/nvidia.go delete mode 100644 pkg/agent/containerd/nvidia_test.go delete mode 100644 pkg/agent/containerd/runwasi.go delete mode 100644 pkg/agent/containerd/runwasi_test.go diff --git a/manifests/runtimes.yaml b/manifests/runtimes.yaml new file mode 100644 index 0000000000..fdbc951228 --- /dev/null +++ b/manifests/runtimes.yaml @@ -0,0 +1,59 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia-experimental +handler: nvidia-experimental +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: crun +handler: crun +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: lunatic +handler: lunatic +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: slight +handler: slight +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: spin +handler: spin +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wws +handler: wws +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmedge +handler: wasmedge +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmer +handler: wasmer +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: wasmtime +handler: wasmtime \ No newline at end of file diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index 354576a685..4a5cd21dfe 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -60,9 +60,12 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { cfg.AgentConfig.Systemd = !isRunningInUserNS && controllers["cpuset"] && os.Getenv("INVOCATION_ID") != "" } - extraRuntimes := runtimeConfigs{} - findNvidiaContainerRuntimes(os.DirFS(string(os.PathSeparator)), extraRuntimes) - findWasiRuntimes(os.DirFS(string(os.PathSeparator)), extraRuntimes) + extraRuntimes := findContainerRuntimes(os.DirFS(string(os.PathSeparator))) + + // Verifies if the DefaultRuntime can be found + if _, ok := extraRuntimes[cfg.DefaultRuntime]; !ok && cfg.DefaultRuntime != "" { + return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime) + } var containerdTemplate string containerdConfig := templates.ContainerdConfig{ diff --git a/pkg/agent/containerd/nvidia.go b/pkg/agent/containerd/nvidia.go deleted file mode 100644 index 372ebdf4f6..0000000000 --- a/pkg/agent/containerd/nvidia.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build linux -// +build linux - -package containerd - -import ( - "io/fs" -) - -// findNvidiaContainerRuntimes returns a list of nvidia container runtimes that -// are available on the system. It checks install locations used by the nvidia -// gpu operator and by system package managers. The gpu operator installation -// takes precedence over the system package manager installation. -// The given fs.FS should represent the filesystem root directory to search in. -func findNvidiaContainerRuntimes(root fs.FS, foundRuntimes runtimeConfigs) { - // Check these locations in order. The GPU operator's installation should - // take precedence over the package manager's installation. - locationsToCheck := []string{ - "usr/local/nvidia/toolkit", // Path when installing via GPU Operator - "usr/bin", // Path when installing via package manager - } - - // Fill in the binary location with just the name of the binary, - // and check against each of the possible locations. If a match is found, - // set the location to the full path. - potentialRuntimes := runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "nvidia-container-runtime-experimental", - }, - } - findContainerRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) -} diff --git a/pkg/agent/containerd/nvidia_test.go b/pkg/agent/containerd/nvidia_test.go deleted file mode 100644 index ebbe08c76b..0000000000 --- a/pkg/agent/containerd/nvidia_test.go +++ /dev/null @@ -1,267 +0,0 @@ -//go:build linux -// +build linux - -package containerd - -import ( - "io/fs" - "reflect" - "testing" - "testing/fstest" -) - -func Test_UnitFindNvidiaContainerRuntimes(t *testing.T) { - executable := &fstest.MapFile{Mode: 0755} - type args struct { - root fs.FS - alreadyFoundRuntimes runtimeConfigs - } - tests := []struct { - name string - args args - want runtimeConfigs - }{ - { - name: "No runtimes", - args: args{ - root: fstest.MapFS{}, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{}, - }, - { - name: "Nvidia runtime in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - }, - }, - { - name: "Experimental runtime in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - }, - }, - { - name: "Two runtimes in separate directories", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - }, - }, - { - name: "Experimental runtime in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Same runtime in two directories", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/bin/nvidia-container-runtime": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in both directories", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - "usr/bin/nvidia-container-runtime": executable, - "usr/bin/nvidia-container-runtime-experimental": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Both runtimes in /usr/bin and one duplicate in /usr/local/nvidia/toolkit", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/bin/nvidia-container-runtime-experimental": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - "nvidia-experimental": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", - }, - }, - }, - { - name: "Runtime is a directory", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": &fstest.MapFile{ - Mode: fs.ModeDir, - }, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{}, - }, - { - name: "Runtime in both directories, but one is a directory", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ - Mode: fs.ModeDir, - }, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - }, - }, - { - name: "Preserve already found runtimes", - args: args{ - root: fstest.MapFS{ - "usr/bin/nvidia-container-runtime": executable, - "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ - Mode: fs.ModeDir, - }, - }, - alreadyFoundRuntimes: runtimeConfigs{ - "slight": { - RuntimeType: "io.containerd.slight.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", - }, - "wasmtime": { - RuntimeType: "io.containerd.wasmtime.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", - }, - }, - }, - want: runtimeConfigs{ - "slight": { - RuntimeType: "io.containerd.slight.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", - }, - "wasmtime": { - RuntimeType: "io.containerd.wasmtime.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", - }, - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - foundRuntimes := tt.args.alreadyFoundRuntimes - findNvidiaContainerRuntimes(tt.args.root, foundRuntimes) - if !reflect.DeepEqual(foundRuntimes, tt.want) { - t.Errorf("findNvidiaContainerRuntimes() = %+v\nWant = %+v", foundRuntimes, tt.want) - } - }) - } -} diff --git a/pkg/agent/containerd/runtimes.go b/pkg/agent/containerd/runtimes.go index a4a09efb34..d946d5c27a 100644 --- a/pkg/agent/containerd/runtimes.go +++ b/pkg/agent/containerd/runtimes.go @@ -16,20 +16,17 @@ import ( // The key holds the name of the runtime type runtimeConfigs map[string]templates.ContainerdRuntimeConfig -// findContainerRuntimes returns a list of container runtimes that -// are available on the system. It checks install locations provided via -// the potentialRuntimes variable. +// searchForRuntimes searches for runtimes and add into foundRuntimes +// It checks install locations provided via potentitalRuntimes variable. // The binaries are searched at the locations specivied by locationsToCheck. -// Note: check the given locations in order. // The given fs.FS should represent the filesystem root directory to search in. -func findContainerRuntimes(root fs.FS, potentialRuntimes runtimeConfigs, locationsToCheck []string, foundRuntimes runtimeConfigs) { +func searchForRuntimes(root fs.FS, potentialRuntimes runtimeConfigs, locationsToCheck []string, foundRuntimes runtimeConfigs) { // Check these locations in order. The GPU operator's installation should // take precedence over the package manager's installation. // Fill in the binary location with just the name of the binary, // and check against each of the possible locations. If a match is found, // set the location to the full path. -RUNTIME: for runtimeName, runtimeConfig := range potentialRuntimes { for _, location := range locationsToCheck { binaryPath := filepath.Join(location, runtimeConfig.BinaryName) @@ -42,8 +39,7 @@ RUNTIME: runtimeConfig.BinaryName = filepath.Join("/", binaryPath) logrus.Infof("Found %s container runtime at %s", runtimeName, runtimeConfig.BinaryName) foundRuntimes[runtimeName] = runtimeConfig - // Skip to the next runtime to enforce precedence. - continue RUNTIME + break } else { if errors.Is(err, fs.ErrNotExist) { logrus.Debugf("%s container runtime not found at /%s", runtimeName, binaryPath) @@ -54,3 +50,112 @@ RUNTIME: } } } + +// findContainerRuntimes is a function that searches for all the runtimes and +// return a list with all the runtimes that have been found +func findContainerRuntimes(root fs.FS) runtimeConfigs { + foundRuntimes := runtimeConfigs{} + findCRunContainerRuntime(root, foundRuntimes) + findNvidiaContainerRuntimes(root, foundRuntimes) + findWasiRuntimes(root, foundRuntimes) + return foundRuntimes +} + +// findCRunContainerRuntime finds if crun is available in the system and adds to foundRuntimes +func findCRunContainerRuntime(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. + locationsToCheck := []string{ + "usr/sbin", // Path when installing via package manager + "usr/bin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "crun": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "crun", + }, + } + + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} + +// findNvidiaContainerRuntimes finds the nvidia runtimes that are are available on the system +// and adds to foundRuntimes. It checks install locations used by the nvidia +// gpu operator and by system package managers. The gpu operator installation +// takes precedence over the system package manager installation. +// The given fs.FS should represent the filesystem root directory to search in. +func findNvidiaContainerRuntimes(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. The GPU operator's installation should + // take precedence over the package manager's installation. + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path when installing via GPU Operator + "usr/bin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime-experimental", + }, + } + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} + +// findWasiRuntimes finds the WebAssembly (WASI) container runtimes that +// are available on the system and adds to foundRuntimes. It checks install locations used by the kwasm +// operator and by system package managers. The kwasm operator installation +// takes precedence over the system package manager installation. +// The given fs.FS should represent the filesystem root directory to search in. +func findWasiRuntimes(root fs.FS, foundRuntimes runtimeConfigs) { + // Check these locations in order. + locationsToCheck := []string{ + "opt/kwasm/bin", // Path when installing via kwasm Operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + // Fill in the binary location with just the name of the binary, + // and check against each of the possible locations. If a match is found, + // set the location to the full path. + potentialRuntimes := runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "containerd-shim-wasmer-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + } + searchForRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) +} diff --git a/pkg/agent/containerd/runtimes_test.go b/pkg/agent/containerd/runtimes_test.go index c5c63af2a6..b209b495d1 100644 --- a/pkg/agent/containerd/runtimes_test.go +++ b/pkg/agent/containerd/runtimes_test.go @@ -12,6 +12,60 @@ import ( func Test_UnitFindContainerRuntimes(t *testing.T) { executable := &fstest.MapFile{Mode: 0755} + + type args struct { + root fs.FS + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + }, + want: runtimeConfigs{}, + }, + { + name: "Found crun, nvidia and wasm", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/crun": executable, + "opt/kwasm/bin/containerd-shim-lunatic-v1": executable, + }, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "crun": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/crun", + }, + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-lunatic-v1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := findContainerRuntimes(tt.args.root) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("findContainerRuntimes = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} + +func Test_UnitSearchContainerRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} locationsToCheck := []string{ "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator @@ -205,10 +259,538 @@ func Test_UnitFindContainerRuntimes(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { foundRuntimes := runtimeConfigs{} - findContainerRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) if !reflect.DeepEqual(foundRuntimes, tt.want) { t.Errorf("findContainerRuntimes() = %+v\nWant = %+v", foundRuntimes, tt.want) } }) } } + +func Test_UnitSearchWasiRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator + "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + potentialRuntimes := runtimeConfigs{ + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "containerd-shim-wasmer-v1", + }, + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + } + + type args struct { + root fs.FS + potentialRuntimes runtimeConfigs + locationsToCheck []string + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{}, + }, + { + name: "wasmtime runtime in /usr/sbin", + args: args{ + root: fstest.MapFS{ + "usr/sbin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/usr/sbin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "lunatic runtime in /opt/kwasm/bin/", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-lunatic-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-lunatic-v1", + }, + }, + }, + { + name: "Two runtimes in separate directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-wasmer-v1": executable, + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "/usr/bin/containerd-shim-wasmer-v1", + }, + }, + }, + { + name: "Same runtime in two directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-wasmedge-v1": executable, + "opt/kwasm/bin/containerd-shim-wasmedge-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmedge-v1", + }, + }, + }, + { + name: "All runtimes in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/containerd-shim-lunatic-v1": executable, + "usr/bin/containerd-shim-slight-v1": executable, + "usr/bin/containerd-shim-spin-v1": executable, + "usr/bin/containerd-shim-wws-v1": executable, + "usr/bin/containerd-shim-wasmedge-v1": executable, + "usr/bin/containerd-shim-wasmer-v1": executable, + "usr/bin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "lunatic": { + RuntimeType: "io.containerd.lunatic.v2", + BinaryName: "/usr/bin/containerd-shim-lunatic-v1", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/usr/bin/containerd-shim-slight-v1", + }, + "spin": { + RuntimeType: "io.containerd.spin.v2", + BinaryName: "/usr/bin/containerd-shim-spin-v1", + }, + "wws": { + RuntimeType: "io.containerd.wws.v2", + BinaryName: "/usr/bin/containerd-shim-wws-v1", + }, + "wasmedge": { + RuntimeType: "io.containerd.wasmedge.v2", + BinaryName: "/usr/bin/containerd-shim-wasmedge-v1", + }, + "wasmer": { + RuntimeType: "io.containerd.wasmer.v2", + BinaryName: "/usr/bin/containerd-shim-wasmer-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/usr/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "Both runtimes in both directories", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "usr/bin/containerd-shim-slight-v1": executable, + "usr/bin/containerd-shim-wasmtime-v1": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + { + name: "Preserve already found runtimes", + args: args{ + root: fstest.MapFS{ + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "usr/bin/nvidia-container-runtime": executable, + }, + locationsToCheck: locationsToCheck, + potentialRuntimes: potentialRuntimes, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := runtimeConfigs{} + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("searchForRuntimes = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} + +func Test_UnitSearchNvidiaContainerRuntimes(t *testing.T) { + executable := &fstest.MapFile{Mode: 0755} + + locationsToCheck := []string{ + "usr/local/nvidia/toolkit", // Path for nvidia shim when installing via GPU Operator + "opt/kwasm/bin", // Path for wasm shim when installing via the kwasm operator + "usr/bin", // Path when installing via package manager + "usr/sbin", // Path when installing via package manager + } + + potentialRuntimes := runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "nvidia-container-runtime-experimental", + }, + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "containerd-shim-wasmtime-v1", + }, + } + + type args struct { + root fs.FS + potentialRuntimes runtimeConfigs + locationsToCheck []string + } + + tests := []struct { + name string + args args + want runtimeConfigs + }{ + { + name: "No runtimes", + args: args{ + root: fstest.MapFS{}, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{}, + }, + { + name: "Nvidia runtime in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + { + name: "Experimental runtime in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + }, + }, + { + name: "Two runtimes in separate directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + }, + }, + { + name: "Experimental runtime in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Same runtime in two directories", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/bin", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/bin/nvidia-container-runtime": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in both directories", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/local/nvidia/toolkit/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Both runtimes in /usr/bin and one duplicate in /usr/local/nvidia/toolkit", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/bin/nvidia-container-runtime-experimental": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime-experimental": executable, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + "nvidia-experimental": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental", + }, + }, + }, + { + name: "Runtime is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{}, + }, + { + name: "Runtime in both directories, but one is a directory", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + { + name: "Preserve already found runtimes", + args: args{ + root: fstest.MapFS{ + "usr/bin/nvidia-container-runtime": executable, + "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, + "opt/kwasm/bin/containerd-shim-slight-v1": executable, + "usr/local/nvidia/toolkit/nvidia-container-runtime": &fstest.MapFile{ + Mode: fs.ModeDir, + }, + }, + potentialRuntimes: potentialRuntimes, + locationsToCheck: locationsToCheck, + }, + want: runtimeConfigs{ + "slight": { + RuntimeType: "io.containerd.slight.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", + }, + "wasmtime": { + RuntimeType: "io.containerd.wasmtime.v2", + BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", + }, + "nvidia": { + RuntimeType: "io.containerd.runc.v2", + BinaryName: "/usr/bin/nvidia-container-runtime", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + foundRuntimes := runtimeConfigs{} + searchForRuntimes(tt.args.root, tt.args.potentialRuntimes, tt.args.locationsToCheck, foundRuntimes) + if !reflect.DeepEqual(foundRuntimes, tt.want) { + t.Errorf("searchForRuntimes() = %+v\nWant = %+v", foundRuntimes, tt.want) + } + }) + } +} diff --git a/pkg/agent/containerd/runwasi.go b/pkg/agent/containerd/runwasi.go deleted file mode 100644 index e64d35c732..0000000000 --- a/pkg/agent/containerd/runwasi.go +++ /dev/null @@ -1,60 +0,0 @@ -//go:build linux -// +build linux - -package containerd - -import ( - "io/fs" -) - -// findWasiRuntimes returns a list of WebAssembly (WASI) container runtimes that -// are available on the system. It checks install locations used by the kwasm -// operator and by system package managers. The kwasm operator installation -// takes precedence over the system package manager installation. -// The given fs.FS should represent the filesystem root directory to search in. -func findWasiRuntimes(root fs.FS, - foundRuntimes runtimeConfigs, -) { - // Check these locations in order. The GPU operator's installation should - // take precedence over the package manager's installation. - locationsToCheck := []string{ - "opt/kwasm/bin", // Path when installing via kwasm Operator - "usr/bin", // Path when installing via package manager - "usr/sbin", // Path when installing via package manager - } - - // Fill in the binary location with just the name of the binary, - // and check against each of the possible locations. If a match is found, - // set the location to the full path. - potentialRuntimes := runtimeConfigs{ - "lunatic": { - RuntimeType: "io.containerd.lunatic.v2", - BinaryName: "containerd-shim-lunatic-v1", - }, - "slight": { - RuntimeType: "io.containerd.slight.v2", - BinaryName: "containerd-shim-slight-v1", - }, - "spin": { - RuntimeType: "io.containerd.spin.v2", - BinaryName: "containerd-shim-spin-v1", - }, - "wws": { - RuntimeType: "io.containerd.wws.v2", - BinaryName: "containerd-shim-wws-v1", - }, - "wasmedge": { - RuntimeType: "io.containerd.wasmedge.v2", - BinaryName: "containerd-shim-wasmedge-v1", - }, - "wasmer": { - RuntimeType: "io.containerd.wasmer.v2", - BinaryName: "containerd-shim-wasmer-v1", - }, - "wasmtime": { - RuntimeType: "io.containerd.wasmtime.v2", - BinaryName: "containerd-shim-wasmtime-v1", - }, - } - findContainerRuntimes(root, potentialRuntimes, locationsToCheck, foundRuntimes) -} diff --git a/pkg/agent/containerd/runwasi_test.go b/pkg/agent/containerd/runwasi_test.go deleted file mode 100644 index d5d5a7a344..0000000000 --- a/pkg/agent/containerd/runwasi_test.go +++ /dev/null @@ -1,199 +0,0 @@ -//go:build linux -// +build linux - -package containerd - -import ( - "io/fs" - "reflect" - "testing" - "testing/fstest" -) - -func Test_UnitFindWasiRuntimes(t *testing.T) { - executable := &fstest.MapFile{Mode: 0755} - type args struct { - root fs.FS - alreadyFoundRuntimes runtimeConfigs - } - tests := []struct { - name string - args args - want runtimeConfigs - }{ - { - name: "No runtimes", - args: args{ - root: fstest.MapFS{}, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{}, - }, - { - name: "wasmtime runtime in /usr/sbin", - args: args{ - root: fstest.MapFS{ - "usr/sbin/containerd-shim-wasmtime-v1": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "wasmtime": { - RuntimeType: "io.containerd.wasmtime.v2", - BinaryName: "/usr/sbin/containerd-shim-wasmtime-v1", - }, - }, - }, - { - name: "lunatic runtime in /opt/kwasm/bin/", - args: args{ - root: fstest.MapFS{ - "opt/kwasm/bin/containerd-shim-lunatic-v1": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "lunatic": { - RuntimeType: "io.containerd.lunatic.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-lunatic-v1", - }, - }, - }, - { - name: "Two runtimes in separate directories", - args: args{ - root: fstest.MapFS{ - "usr/bin/containerd-shim-wasmer-v1": executable, - "opt/kwasm/bin/containerd-shim-slight-v1": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "slight": { - RuntimeType: "io.containerd.slight.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", - }, - "wasmer": { - RuntimeType: "io.containerd.wasmer.v2", - BinaryName: "/usr/bin/containerd-shim-wasmer-v1", - }, - }, - }, - { - name: "Same runtime in two directories", - args: args{ - root: fstest.MapFS{ - "usr/bin/containerd-shim-wasmedge-v1": executable, - "opt/kwasm/bin/containerd-shim-wasmedge-v1": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "wasmedge": { - RuntimeType: "io.containerd.wasmedge.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-wasmedge-v1", - }, - }, - }, - { - name: "All runtimes in /usr/bin", - args: args{ - root: fstest.MapFS{ - "usr/bin/containerd-shim-lunatic-v1": executable, - "usr/bin/containerd-shim-slight-v1": executable, - "usr/bin/containerd-shim-spin-v1": executable, - "usr/bin/containerd-shim-wws-v1": executable, - "usr/bin/containerd-shim-wasmedge-v1": executable, - "usr/bin/containerd-shim-wasmer-v1": executable, - "usr/bin/containerd-shim-wasmtime-v1": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "lunatic": { - RuntimeType: "io.containerd.lunatic.v2", - BinaryName: "/usr/bin/containerd-shim-lunatic-v1", - }, - "slight": { - RuntimeType: "io.containerd.slight.v2", - BinaryName: "/usr/bin/containerd-shim-slight-v1", - }, - "spin": { - RuntimeType: "io.containerd.spin.v2", - BinaryName: "/usr/bin/containerd-shim-spin-v1", - }, - "wws": { - RuntimeType: "io.containerd.wws.v2", - BinaryName: "/usr/bin/containerd-shim-wws-v1", - }, - "wasmedge": { - RuntimeType: "io.containerd.wasmedge.v2", - BinaryName: "/usr/bin/containerd-shim-wasmedge-v1", - }, - "wasmer": { - RuntimeType: "io.containerd.wasmer.v2", - BinaryName: "/usr/bin/containerd-shim-wasmer-v1", - }, - "wasmtime": { - RuntimeType: "io.containerd.wasmtime.v2", - BinaryName: "/usr/bin/containerd-shim-wasmtime-v1", - }, - }, - }, - { - name: "Both runtimes in both directories", - args: args{ - root: fstest.MapFS{ - "opt/kwasm/bin/containerd-shim-slight-v1": executable, - "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, - "usr/bin/containerd-shim-slight-v1": executable, - "usr/bin/containerd-shim-wasmtime-v1": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{}, - }, - want: runtimeConfigs{ - "slight": { - RuntimeType: "io.containerd.slight.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-slight-v1", - }, - "wasmtime": { - RuntimeType: "io.containerd.wasmtime.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", - }, - }, - }, - { - name: "Preserve already found runtimes", - args: args{ - root: fstest.MapFS{ - "opt/kwasm/bin/containerd-shim-wasmtime-v1": executable, - }, - alreadyFoundRuntimes: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - }, - }, - want: runtimeConfigs{ - "nvidia": { - RuntimeType: "io.containerd.runc.v2", - BinaryName: "/usr/bin/nvidia-container-runtime", - }, - "wasmtime": { - RuntimeType: "io.containerd.wasmtime.v2", - BinaryName: "/opt/kwasm/bin/containerd-shim-wasmtime-v1", - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - foundRuntimes := tt.args.alreadyFoundRuntimes - findWasiRuntimes(tt.args.root, foundRuntimes) - if !reflect.DeepEqual(foundRuntimes, tt.want) { - t.Errorf("findWasiRuntimes() = %+v\nWant = %+v", foundRuntimes, tt.want) - } - }) - } -} diff --git a/pkg/agent/run_linux.go b/pkg/agent/run_linux.go index caa4c2bae4..32aa9dd92c 100644 --- a/pkg/agent/run_linux.go +++ b/pkg/agent/run_linux.go @@ -36,6 +36,11 @@ func setupCriCtlConfig(cfg cmds.Agent, nodeConfig *config.Node) error { } } + // Send to node struct the value from cli/config default runtime + if cfg.DefaultRuntime != "" { + nodeConfig.DefaultRuntime = cfg.DefaultRuntime + } + crp := "runtime-endpoint: " + cre + "\n" ise := nodeConfig.ImageServiceEndpoint if ise != "" && ise != cre { diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 002f806caa..66f8ee1080 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -36,6 +36,7 @@ version = 2 [plugins."io.containerd.grpc.v1.cri".containerd] snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} + {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}} {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} {{ if .NodeConfig.AgentConfig.ImageServiceSocket }} [plugins."io.containerd.snapshotter.v1.stargz"] diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 29dce58574..b36f22fe9d 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -27,6 +27,7 @@ type Agent struct { Snapshotter string Docker bool ContainerRuntimeEndpoint string + DefaultRuntime string ImageServiceEndpoint string FlannelIface string FlannelConf string @@ -115,6 +116,11 @@ var ( Usage: "(agent/runtime) Disable embedded containerd and use the CRI socket at the given path; when used with --docker this sets the docker socket path", Destination: &AgentConfig.ContainerRuntimeEndpoint, } + DefaultRuntimeFlag = &cli.StringFlag{ + Name: "default-runtime", + Usage: "(agent/runtime) Set the default runtime in containerd", + Destination: &AgentConfig.DefaultRuntime, + } ImageServiceEndpointFlag = &cli.StringFlag{ Name: "image-service-endpoint", Usage: "(agent/runtime) Disable embedded containerd image service and use remote image service socket at the given path. If not specified, defaults to --container-runtime-endpoint.", @@ -258,6 +264,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { LBServerPortFlag, ProtectKernelDefaultsFlag, CRIEndpointFlag, + DefaultRuntimeFlag, ImageServiceEndpointFlag, PauseImageFlag, SnapshotterFlag, diff --git a/pkg/cli/cmds/stage.go b/pkg/cli/cmds/stage.go index 32193ca526..1822cdb776 100644 --- a/pkg/cli/cmds/stage.go +++ b/pkg/cli/cmds/stage.go @@ -6,5 +6,5 @@ const ( // coredns and servicelb run controllers that are turned off when their manifests are disabled. // The k3s CloudController also has a bundled manifest and can be disabled via the // --disable-cloud-controller flag or --disable=ccm, but the latter method is not documented. - DisableItems = "coredns, servicelb, traefik, local-storage, metrics-server" + DisableItems = "coredns, servicelb, traefik, local-storage, metrics-server, runtimes" ) diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 4609b7a5ce..e91cdcd11e 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -55,6 +55,7 @@ type Node struct { Token string Certificate *tls.Certificate ServerHTTPSPort int + DefaultRuntime string } type Containerd struct { diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index 52d1c08d8c..91ac5c4236 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -11,6 +11,7 @@ // manifests/metrics-server/metrics-server-service.yaml // manifests/metrics-server/resource-reader.yaml // manifests/rolebindings.yaml +// manifests/runtimes.yaml // manifests/traefik.yaml //go:build !no_stage // +build !no_stage @@ -311,6 +312,26 @@ func rolebindingsYaml() (*asset, error) { return a, nil } +var _runtimesYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\xd0\x31\x8e\x84\x30\x0c\x05\xd0\x3e\xa7\xc8\x05\xc2\x6a\xbb\x55\xda\xbd\xc1\x14\xd3\x5b\xc4\x02\x8b\xc4\xa0\x38\xc0\x1c\x7f\x04\x1a\x26\x40\xed\xf2\x7f\x4b\xef\x4b\x86\x89\x9e\x98\x85\x46\xf6\x96\xc7\x80\xcd\xf0\x27\x0d\x8d\x3f\xcb\xaf\x19\x88\x83\xb7\x8f\x99\x0b\x25\xfc\x8f\x20\x62\x12\x16\x08\x50\xc0\x1b\x6b\x19\x12\x7a\xcb\x0b\x05\x02\xd3\x03\x87\x88\xf9\x9b\x9d\x73\x46\x87\x76\xf8\x9a\x30\x53\x42\x2e\x10\xef\x3b\xd7\xa3\xc6\x68\x9b\x67\xae\x2b\x7b\xd2\x60\xe3\xcc\x50\xa8\xad\xf2\x51\x68\xe0\x12\xa9\xeb\x4b\xb5\x3f\x59\x85\x9e\xe8\xf4\x8e\x3d\x69\xb0\xeb\x2a\x55\xdd\x82\x0a\x0a\x92\x30\x74\x78\x92\x8f\x46\x8d\xcf\x37\x3c\xab\xd1\xdb\xfd\x8a\x6f\xcd\x3b\x00\x00\xff\xff\xc7\xad\x48\x21\x9f\x03\x00\x00") + +func runtimesYamlBytes() ([]byte, error) { + return bindataRead( + _runtimesYaml, + "runtimes.yaml", + ) +} + +func runtimesYaml() (*asset, error) { + bytes, err := runtimesYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "runtimes.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _traefikYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x91\x5f\x6b\xdb\x4a\x10\xc5\xdf\xf5\x29\x06\x81\x9f\x2e\x2b\x39\x86\x40\xd0\x9b\xaf\xa3\xdc\x1b\xee\x6d\x1a\x2c\xa7\x25\x4f\x66\xbc\x1a\x5b\x8b\x57\xbb\xcb\xec\xc8\xd4\x4d\xf3\xdd\xcb\xda\xce\x3f\x08\xb4\x94\xf6\x6d\xd9\x99\xf9\x9d\x99\x73\x94\x52\x19\x06\xf3\x89\x38\x1a\xef\x2a\xe8\xc8\xf6\x85\x46\x11\x4b\x85\xf1\xe5\xee\x2c\xdb\x1a\xd7\x56\xf0\x2f\xd9\x7e\xd6\x21\x4b\xd6\x93\x60\x8b\x82\x55\x06\xe0\xb0\xa7\x0a\x84\x91\xd6\x66\xab\x34\xb7\xa7\xbf\x18\x50\x53\x05\xdb\x61\x45\x2a\xee\xa3\x50\x9f\xc5\x40\x3a\x8d\xe8\x04\xa9\xa0\x13\x09\xb1\x2a\xcb\xd1\xc3\x7f\x77\x7f\xd7\xf3\x9b\x7a\x51\x37\xcb\xe9\xed\xf5\xe3\xa8\x8c\x82\x62\x74\x79\x68\x8c\xe5\x2b\xb8\x9a\x9c\x17\xe3\x62\xf2\xd7\x10\x0e\x8f\x71\x21\x9b\xaf\xd9\x6f\x3c\xe0\xcf\x2d\xff\xde\xe2\x00\x91\x24\x41\x01\x36\xd6\xaf\xd0\x16\x47\xb1\x4b\x5a\xe3\x60\x65\x4e\x1b\x13\x85\xf7\x15\xe4\xa3\x87\xe6\xbe\x59\xd4\x1f\x96\x97\xf5\xd5\xf4\xee\xff\xc5\x72\x5e\xff\x73\xdd\x2c\xe6\xf7\xcb\xf9\xf4\xf3\xe3\x28\xcf\x00\x76\x68\x07\x8a\x33\xef\x84\x9c\x54\xf0\x4d\x1d\xb8\xc1\xb7\x53\xe7\x7c\x5a\xc9\xbb\x78\xd4\x02\x08\xec\x7b\x92\x8e\x86\x98\x0c\x0a\x3e\x5d\x94\x5f\x8c\x2f\x26\xf9\xbb\x0d\x51\x33\x06\xaa\x20\x17\x1e\xe8\xd8\x12\xd8\xef\x4c\x4b\xfc\x8c\x4c\x5e\xb1\x23\xa1\x78\xed\x36\x4c\xf1\xb9\x00\x10\x86\x95\x35\xb1\xa3\xb6\x21\xde\x19\x4d\x2f\x15\x00\x72\xb8\xb2\xd4\xa6\x00\x06\x3a\x91\x8d\x67\x23\xfb\x99\xc5\x18\x6f\x0e\xe1\xe4\x47\x5b\x94\xb6\x43\x14\x62\xa5\xd9\x88\xd1\x68\x8f\xab\x98\x1e\x37\xcf\x4c\xa6\xe0\xa3\x11\x7f\x70\x8d\xd1\xe9\x8e\xb8\xec\x0d\xb3\x67\x6a\x95\x35\x2b\x46\xde\xab\x53\x28\x4f\xd7\x0a\x6e\x2a\xc8\x27\xc5\xd9\xb8\x38\x3f\xfe\x89\xb7\xc4\xaf\x3d\x53\xb0\xa5\x84\x9c\x9d\xa4\xa7\x6d\xeb\x5d\xfc\xe8\xec\xfe\x09\xe2\x43\x9a\xf0\x5c\x41\x5e\x7f\x31\x51\x62\xfe\x66\xd0\xf9\x96\x14\x7b\x4b\xc5\x8b\x53\xc9\x5b\xed\x9d\xb0\xb7\x2a\x58\x74\xf4\x03\x16\x00\xad\xd7\xa4\x53\x58\x37\xbe\xd1\x1d\xb5\x83\xa5\x9f\x93\xe9\x31\x39\xf7\xeb\xfc\xf8\x36\x3a\x13\xae\xb0\x37\x76\x7f\xeb\xad\xd1\x49\xf7\x96\x69\x4d\x7c\x39\xa0\x6d\x04\xf5\x36\xcf\xbe\x07\x00\x00\xff\xff\x19\x56\x7e\xa9\x56\x04\x00\x00") func traefikYamlBytes() ([]byte, error) { @@ -394,6 +415,7 @@ var _bindata = map[string]func() (*asset, error){ "metrics-server/metrics-server-service.yaml": metricsServerMetricsServerServiceYaml, "metrics-server/resource-reader.yaml": metricsServerResourceReaderYaml, "rolebindings.yaml": rolebindingsYaml, + "runtimes.yaml": runtimesYaml, "traefik.yaml": traefikYaml, } @@ -453,6 +475,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "resource-reader.yaml": &bintree{metricsServerResourceReaderYaml, map[string]*bintree{}}, }}, "rolebindings.yaml": &bintree{rolebindingsYaml, map[string]*bintree{}}, + "runtimes.yaml": &bintree{runtimesYaml, map[string]*bintree{}}, "traefik.yaml": &bintree{traefikYaml, map[string]*bintree{}}, }}