Merge pull request #2668 from Oats87/issues/k3s/2548-mst-suppl2

Set kubelet-cgroups if we detect we are running under a `.scope`

pkg/daemons/agent/agent.go

@@ -11,6 +11,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/daemons/executor"
+	"github.com/rancher/k3s/pkg/version"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/component-base/logs"
@@ -120,7 +121,7 @@ func startKubelet(cfg *config.Agent) error {
 	if err != nil || defaultIP.String() != cfg.NodeIP {
 		argsMap["node-ip"] = cfg.NodeIP
 	}
-	root, hasCFS, hasPIDs := checkCgroups()
+	kubeletRoot, runtimeRoot, hasCFS, hasPIDs := checkCgroups()
 	if !hasCFS {
 		logrus.Warn("Disabling CPU quotas due to missing cpu.cfs_period_us")
 		argsMap["cpu-cfs-quota"] = "false"
@@ -131,9 +132,11 @@ func startKubelet(cfg *config.Agent) error {
 		argsMap["enforce-node-allocatable"] = ""
 		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "SupportPodPidsLimit=false")
 	}
-	if root != "" {
+	if kubeletRoot != "" {
-		argsMap["runtime-cgroups"] = root
+		argsMap["kubelet-cgroups"] = kubeletRoot
-		argsMap["kubelet-cgroups"] = root
+	}
+	if runtimeRoot != "" {
+		argsMap["runtime-cgroups"] = runtimeRoot
 	}
 	if system.RunningInUserNS() {
 		argsMap["feature-gates"] = addFeatureGate(argsMap["feature-gates"], "DevicePlugins=false")
@@ -172,10 +175,10 @@ func addFeatureGate(current, new string) string {
 	return current + "," + new
 }
 
-func checkCgroups() (root string, hasCFS bool, hasPIDs bool) {
+func checkCgroups() (kubeletRoot, runtimeRoot string, hasCFS, hasPIDs bool) {
 	f, err := os.Open("/proc/self/cgroup")
 	if err != nil {
-		return "", false, false
+		return "", "", false, false
 	}
 	defer f.Close()
 
@@ -194,37 +197,52 @@ func checkCgroups() (root string, hasCFS bool, hasPIDs bool) {
 				if _, err := os.Stat(p); err == nil {
 					hasCFS = true
 				}
-			}
+			} else if system == "name=systemd" {
-		}
+				// If we detect that we are running under a `.scope` unit with systemd
-	}
+				// we can assume we are being directly invoked from the command line
-
+				// and thus need to set our kubelet root to something out of the context
-	// Examine process ID 1 to see if there is a cgroup assigned to it.
+				// of `/user.slice` to ensure that `CPUAccounting` and `MemoryAccounting`
-	// When we are not in a container, process 1 is likely to be systemd or some other service manager.
+				// are enabled, as they are generally disabled by default for `user.slice`
-	// It either lives at `/` or `/init.scope` according to https://man7.org/linux/man-pages/man7/systemd.special.7.html
+				// Note that we are not setting the `runtimeRoot` as if we are running with
-	// When containerized, process 1 will be generally be in a cgroup, otherwise, we may be running in
+				// `--docker`, we will inadvertently move the cgroup `dockerd` lives in
-	// a host PID scenario but we don't support this.
+				//  which is not ideal and causes dockerd to become unmanageable by systemd.
-	g, err := os.Open("/proc/1/cgroup")
-	if err != nil {
-		return "", false, false
-	}
-	defer g.Close()
-	root = ""
-	scan = bufio.NewScanner(g)
-	for scan.Scan() {
-		parts := strings.Split(scan.Text(), ":")
-		if len(parts) < 3 {
-			continue
-		}
-		systems := strings.Split(parts[1], ",")
-		for _, system := range systems {
-			if system == "name=systemd" {
 				last := parts[len(parts)-1]
-				if last != "/" && last != "/init.scope" {
+				i := strings.LastIndex(last, ".scope")
-					root = "/systemd"
+				if i > 0 {
+					kubeletRoot = "/" + version.Program
 				}
 			}
 		}
 	}
 
-	return root, hasCFS, hasPIDs
+	if kubeletRoot == "" {
+		// Examine process ID 1 to see if there is a cgroup assigned to it.
+		// When we are not in a container, process 1 is likely to be systemd or some other service manager.
+		// It either lives at `/` or `/init.scope` according to https://man7.org/linux/man-pages/man7/systemd.special.7.html
+		// When containerized, process 1 will be generally be in a cgroup, otherwise, we may be running in
+		// a host PID scenario but we don't support this.
+		g, err := os.Open("/proc/1/cgroup")
+		if err != nil {
+			return "", "", false, false
+		}
+		defer g.Close()
+		scan = bufio.NewScanner(g)
+		for scan.Scan() {
+			parts := strings.Split(scan.Text(), ":")
+			if len(parts) < 3 {
+				continue
+			}
+			systems := strings.Split(parts[1], ",")
+			for _, system := range systems {
+				if system == "name=systemd" {
+					last := parts[len(parts)-1]
+					if last != "/" && last != "/init.scope" {
+						kubeletRoot = "/" + version.Program
+						runtimeRoot = "/" + version.Program
+					}
+				}
+			}
+		}
+	}
+	return kubeletRoot, runtimeRoot, hasCFS, hasPIDs
 }