deprecate the "node-role.kubernetes.io/master" label / taint

Related to https://github.com/kubernetes/kubernetes/pull/95382

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

manifests/coredns.yaml

@@ -100,6 +100,9 @@ spec:
       tolerations:
         - key: "CriticalAddonsOnly"
           operator: "Exists"
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
+          effect: "NoSchedule"
         - key: "node-role.kubernetes.io/master"
           operator: "Exists"
           effect: "NoSchedule"

manifests/local-storage.yaml

@@ -55,6 +55,9 @@ spec:
       tolerations:
           - key: "CriticalAddonsOnly"
             operator: "Exists"
+          - key: "node-role.kubernetes.io/control-plane"
+            operator: "Exists"
+            effect: "NoSchedule"
           - key: "node-role.kubernetes.io/master"
             operator: "Exists"
             effect: "NoSchedule"

manifests/metrics-server/metrics-server-deployment.yaml

@@ -27,6 +27,9 @@ spec:
       tolerations:
         - key: "CriticalAddonsOnly"
           operator: "Exists"
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
+          effect: "NoSchedule"
         - key: "node-role.kubernetes.io/master"
           operator: "Exists"
           effect: "NoSchedule"

manifests/traefik.yaml

@@ -21,6 +21,9 @@ spec:
     tolerations:
       - key: "CriticalAddonsOnly"
         operator: "Exists"
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Exists"
+        effect: "NoSchedule"
       - key: "node-role.kubernetes.io/master"
         operator: "Exists"
         effect: "NoSchedule"

pkg/etcd/controller.go

@@ -11,10 +11,10 @@ import (
 )
 
 const (
-	nodeID      = "etcd.k3s.cattle.io/node-name"
+	nodeID       = "etcd.k3s.cattle.io/node-name"
-	nodeAddress = "etcd.k3s.cattle.io/node-address"
+	nodeAddress  = "etcd.k3s.cattle.io/node-address"
-	master      = "node-role.kubernetes.io/master"
+	controlPlane = "node-role.kubernetes.io/control-plane"
-	etcdRole    = "node-role.kubernetes.io/etcd"
+	etcdRole     = "node-role.kubernetes.io/etcd"
 )
 
 type NodeControllerGetter func() controllerv1.NodeController
@@ -58,7 +58,7 @@ func (h *handler) handleSelf(node *v1.Node) (*v1.Node, error) {
 	if node.Annotations[nodeID] == h.etcd.name &&
 		node.Annotations[nodeAddress] == h.etcd.address &&
 		node.Labels[etcdRole] == "true" &&
-		node.Labels[master] == "true" {
+		node.Labels[controlPlane] == "true" {
 		return node, nil
 	}
 
@@ -69,7 +69,7 @@ func (h *handler) handleSelf(node *v1.Node) (*v1.Node, error) {
 	node.Annotations[nodeID] = h.etcd.name
 	node.Annotations[nodeAddress] = h.etcd.address
 	node.Labels[etcdRole] = "true"
-	node.Labels[master] = "true"
+	node.Labels[controlPlane] = "true"
 
 	return h.nodeController.Update(node)
 }

pkg/server/server.go

@@ -37,7 +37,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/net"
 )
 
-const MasterRoleLabelKey = "node-role.kubernetes.io/master"
+const (
+	MasterRoleLabelKey       = "node-role.kubernetes.io/master"
+	ControlPlaneRoleLabelKey = "node-role.kubernetes.io/control-plane"
+)
 
 func resolveDataDir(dataDir string) (string, error) {
 	dataDir, err := datadir.Resolve(dataDir)
@@ -117,7 +120,7 @@ func runControllers(ctx context.Context, config *Config) error {
 	}
 
 	start := func(ctx context.Context) {
-		if err := masterControllers(ctx, sc, config); err != nil {
+		if err := coreControllers(ctx, sc, config); err != nil {
 			panic(err)
 		}
 		if err := sc.Start(ctx); err != nil {
@@ -130,7 +133,7 @@ func runControllers(ctx context.Context, config *Config) error {
 		controlConfig.Runtime.Handler = handler
 	}
 	if !config.DisableAgent {
-		go setMasterRoleLabel(ctx, sc.Core.Core().V1().Node())
+		go setControlPlaneRoleLabel(ctx, sc.Core.Core().V1().Node())
 	}
 
 	go setClusterDNSConfig(ctx, config, sc.Core.Core().V1().ConfigMap())
@@ -148,7 +151,7 @@ func runControllers(ctx context.Context, config *Config) error {
 	return nil
 }
 
-func masterControllers(ctx context.Context, sc *Context, config *Config) error {
+func coreControllers(ctx context.Context, sc *Context, config *Config) error {
 	if err := nodepassword.MigrateFile(
 		sc.Core.Core().V1().Secret(),
 		sc.Core.Core().V1().Node(),
@@ -419,25 +422,26 @@ func isSymlink(config string) bool {
 	return false
 }
 
-func setMasterRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
+func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
 	for {
 		nodeName := os.Getenv("NODE_NAME")
 		node, err := nodes.Get(nodeName, metav1.GetOptions{})
 		if err != nil {
-			logrus.Infof("Waiting for master node %s startup: %v", nodeName, err)
+			logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err)
 			time.Sleep(1 * time.Second)
 			continue
 		}
-		if v, ok := node.Labels[MasterRoleLabelKey]; ok && v == "true" {
+		if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" {
 			break
 		}
 		if node.Labels == nil {
 			node.Labels = make(map[string]string)
 		}
+		node.Labels[ControlPlaneRoleLabelKey] = "true"
 		node.Labels[MasterRoleLabelKey] = "true"
 		_, err = nodes.Update(node)
 		if err == nil {
-			logrus.Infof("Master role label has been set successfully on node: %s", nodeName)
+			logrus.Infof("Control-plane role label has been set successfully on node: %s", nodeName)
 			break
 		}
 		select {
@@ -479,7 +483,7 @@ func setClusterDNSConfig(ctx context.Context, controlConfig *Config, configMap v
 			logrus.Infof("Cluster dns configmap has been set successfully")
 			break
 		}
-		logrus.Infof("Waiting for master node %s startup: %v", nodeName, err)
+		logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err)
 
 		select {
 		case <-ctx.Done():

pkg/servicelb/controller.go

@@ -362,12 +362,20 @@ func (h *handler) newDaemonSet(svc *core.Service) (*apps.DaemonSet, error) {
 	}
 
 	// Add toleration to noderole.kubernetes.io/master=*:NoSchedule
-	noScheduleToleration := core.Toleration{
+	masterToleration := core.Toleration{
 		Key:      "node-role.kubernetes.io/master",
 		Operator: "Exists",
 		Effect:   "NoSchedule",
 	}
-	ds.Spec.Template.Spec.Tolerations = append(ds.Spec.Template.Spec.Tolerations, noScheduleToleration)
+	ds.Spec.Template.Spec.Tolerations = append(ds.Spec.Template.Spec.Tolerations, masterToleration)
+
+	// Add toleration to noderole.kubernetes.io/control-plane=*:NoSchedule
+	controlPlaneToleration := core.Toleration{
+		Key:      "node-role.kubernetes.io/control-plane",
+		Operator: "Exists",
+		Effect:   "NoSchedule",
+	}
+	ds.Spec.Template.Spec.Tolerations = append(ds.Spec.Template.Spec.Tolerations, controlPlaneToleration)
 
 	// Add toleration to CriticalAddonsOnly
 	criticalAddonsOnlyToleration := core.Toleration{