k3s/pkg/etcd/member_controller.go
Brad Davidson f37e7565b8 Move the apiserver addresses controller into the etcd package
This controller only needs to run when using managed etcd, so move it in
with the rest of the etcd stuff. This change also modifies the
controller to only watch the Kubernetes service endpoint, instead of
watching all endpoints in the entire cluster.

Fixes an error message revealed by use of a newer grpc client in
Kubernetes 1.24, which logs an error when the Put to etcd failed because
kine doesn't support the etcd Put operation. The controller shouldn't
have been running without etcd in the first place.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
2022-04-07 11:28:15 -07:00

132 lines
4.8 KiB
Go

package etcd
import (
"context"
"fmt"
"strings"
"github.com/k3s-io/k3s/pkg/version"
controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
"github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
)
func registerMemberHandlers(ctx context.Context, etcd *ETCD) {
if etcd.config.DisableETCD {
return
}
nodes := etcd.config.Runtime.Core.Core().V1().Node()
e := &etcdMemberHandler{
etcd: etcd,
nodeController: nodes,
ctx: ctx,
}
logrus.Infof("Starting managed etcd member removal controller")
nodes.OnChange(ctx, "managed-etcd-controller", e.sync)
nodes.OnRemove(ctx, "managed-etcd-controller", e.onRemove)
}
var (
removalAnnotation = "etcd." + version.Program + ".cattle.io/remove"
removedNodeNameAnnotation = "etcd." + version.Program + ".cattle.io/removed-node-name"
)
type etcdMemberHandler struct {
etcd *ETCD
nodeController controllerv1.NodeController
ctx context.Context
}
func (e *etcdMemberHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
if node == nil {
return nil, nil
}
if _, ok := node.Labels[EtcdRoleLabel]; !ok {
logrus.Debugf("Node %s was not labeled etcd node, skipping sync", key)
return node, nil
}
node = node.DeepCopy()
if removalRequested, ok := node.Annotations[removalAnnotation]; ok {
if removed, ok := node.Annotations[removedNodeNameAnnotation]; ok {
// check to see if removed is true. if it is, nothing to do.
if currentNodeName, ok := node.Annotations[NodeNameAnnotation]; ok {
if currentNodeName != removed {
// If the current node name is not the same as the removed node name, reset the tainted annotation and removed node name
logrus.Infof("Resetting removed node flag as removed node name (did not match current node name")
delete(node.Annotations, removedNodeNameAnnotation)
node.Annotations[removalAnnotation] = "false"
return e.nodeController.Update(node)
}
// this is the case where the current node name matches the removed node name. We have already removed the
// node, so no need to perform any action. Fallthrough to the non-op below.
}
// This is the edge case where the removed annotation exists, but there is not a current node name annotation.
// This should be a non-op, as we can't remove the node anyway.
logrus.Debugf("etcd member %s was already marked via annotations as removed", key)
return node, nil
}
if strings.ToLower(removalRequested) == "true" {
// remove the member.
name, ok := node.Annotations[NodeNameAnnotation]
if !ok {
return node, fmt.Errorf("node name annotation for node %s not found", key)
}
address, ok := node.Annotations[NodeAddressAnnotation]
if !ok {
return node, fmt.Errorf("node address annotation for node %s not found", key)
}
logrus.Debugf("removing etcd member from cluster name: %s address: %s", name, address)
if err := e.etcd.RemovePeer(e.ctx, name, address, true); err != nil {
return node, err
}
logrus.Debugf("etcd member removal successful for name: %s address: %s", name, address)
// Set the removed node name annotation and clean up the other etcd node annotations.
// These will be set if the tombstone file is then created and the etcd member is re-added, to their new
// respective values.
node.Annotations[removedNodeNameAnnotation] = name
delete(node.Annotations, NodeNameAnnotation)
delete(node.Annotations, NodeAddressAnnotation)
return e.nodeController.Update(node)
}
// In the event that we had an unexpected removal value, simply return.
// Fallthrough to the non-op below.
}
// This is a non-op, as we don't have a tainted annotation to worry about.
return node, nil
}
func (e *etcdMemberHandler) onRemove(key string, node *v1.Node) (*v1.Node, error) {
if _, ok := node.Labels[EtcdRoleLabel]; !ok {
logrus.Debugf("Node %s was not labeled etcd node, skipping etcd member removal", key)
return node, nil
}
logrus.Infof("Removing etcd member %s from cluster", key)
if removalRequested, ok := node.Annotations[removalAnnotation]; ok {
if strings.ToLower(removalRequested) == "true" {
if removedNodeName, ok := node.Annotations[removedNodeNameAnnotation]; ok {
if len(removedNodeName) > 0 {
// If we received a node to delete that has already been removed via annotation, it will be missing
// the corresponding node name and address annotations.
logrus.Infof("etcd member %s was already removed as member name %s via annotation from the cluster", key, removedNodeName)
return node, nil
}
}
}
}
name, ok := node.Annotations[NodeNameAnnotation]
if !ok {
return node, fmt.Errorf("node name annotation for node %s not found", key)
}
address, ok := node.Annotations[NodeAddressAnnotation]
if !ok {
return node, fmt.Errorf("node address annotation for node %s not found", key)
}
return node, e.etcd.RemovePeer(e.ctx, name, address, true)
}