k3s/pkg/etcd/member_controller.go

package etcd

import (
	"context"
	"fmt"
	"strings"

	"github.com/k3s-io/k3s/pkg/version"
	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
)

func RegisterMemberHandlers(ctx context.Context, etcd *ETCD, nodes controllerv1.NodeController) {
	e := &etcdMemberHandler{
		etcd:           etcd,
		nodeController: nodes,
		ctx:            ctx,
	}
	nodes.OnChange(ctx, "managed-etcd-controller", e.sync)
	nodes.OnRemove(ctx, "managed-etcd-controller", e.onRemove)
}

var (
	removalAnnotation         = "etcd." + version.Program + ".cattle.io/remove"
	removedNodeNameAnnotation = "etcd." + version.Program + ".cattle.io/removed-node-name"
)

type etcdMemberHandler struct {
	etcd           *ETCD
	nodeController controllerv1.NodeController
	ctx            context.Context
}

func (e *etcdMemberHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
	if node == nil {
		return nil, nil
	}

	if _, ok := node.Labels[EtcdRoleLabel]; !ok {
		logrus.Debugf("Node %s was not labeled etcd node, skipping sync", key)
		return node, nil
	}

	node = node.DeepCopy()

	if removalRequested, ok := node.Annotations[removalAnnotation]; ok {
		if removed, ok := node.Annotations[removedNodeNameAnnotation]; ok {
			// check to see if removed is true. if it is, nothing to do.
			if currentNodeName, ok := node.Annotations[NodeNameAnnotation]; ok {
				if currentNodeName != removed {
					// If the current node name is not the same as the removed node name, reset the tainted annotation and removed node name
					logrus.Infof("Resetting removed node flag as removed node name ( did not match current node name")
					delete(node.Annotations, removedNodeNameAnnotation)
					node.Annotations[removalAnnotation] = "false"
					return e.nodeController.Update(node)
				}
				// this is the case where the current node name matches the removed node name. We have already removed the
				// node, so no need to perform any action. Fallthrough to the non-op below.
			}
			// This is the edge case where the removed annotation exists, but there is not a current node name annotation.
			// This should be a non-op, as we can't remove the node anyway.
			logrus.Debugf("etcd member %s was already marked via annotations as removed", key)
			return node, nil
		}
		if strings.ToLower(removalRequested) == "true" {
			// remove the member.
			name, ok := node.Annotations[NodeNameAnnotation]
			if !ok {
				return node, fmt.Errorf("node name annotation for node %s not found", key)
			}
			address, ok := node.Annotations[NodeAddressAnnotation]
			if !ok {
				return node, fmt.Errorf("node address annotation for node %s not found", key)
			}

			logrus.Debugf("removing etcd member from cluster name: %s address: %s", name, address)
			if err := e.etcd.RemovePeer(e.ctx, name, address, true); err != nil {
				return node, err
			}
			logrus.Debugf("etcd member removal successful for name: %s address: %s", name, address)
			// Set the removed node name annotation and clean up the other etcd node annotations.
			// These will be set if the tombstone file is then created and the etcd member is re-added, to their new
			// respective values.
			node.Annotations[removedNodeNameAnnotation] = name
			delete(node.Annotations, NodeNameAnnotation)
			delete(node.Annotations, NodeAddressAnnotation)
			return e.nodeController.Update(node)
		}
		// In the event that we had an unexpected removal value, simply return.
		// Fallthrough to the non-op below.
	}
	// This is a non-op, as we don't have a tainted annotation to worry about.
	return node, nil
}

func (e *etcdMemberHandler) onRemove(key string, node *v1.Node) (*v1.Node, error) {
	if _, ok := node.Labels[EtcdRoleLabel]; !ok {
		logrus.Debugf("Node %s was not labeled etcd node, skipping etcd member removal", key)
		return node, nil
	}
	logrus.Infof("Removing etcd member %s from cluster", key)
	if removalRequested, ok := node.Annotations[removalAnnotation]; ok {
		if strings.ToLower(removalRequested) == "true" {
			if removedNodeName, ok := node.Annotations[removedNodeNameAnnotation]; ok {
				if len(removedNodeName) > 0 {
					// If we received a node to delete that has already been removed via annotation, it will be missing
					// the corresponding node name and address annotations.
					logrus.Infof("etcd member %s was already removed as member name %s via annotation from the cluster", key, removedNodeName)
					return node, nil
				}
			}
		}
	}
	name, ok := node.Annotations[NodeNameAnnotation]
	if !ok {
		return node, fmt.Errorf("node name annotation for node %s not found", key)
	}
	address, ok := node.Annotations[NodeAddressAnnotation]
	if !ok {
		return node, fmt.Errorf("node address annotation for node %s not found", key)
	}
	return node, e.etcd.RemovePeer(e.ctx, name, address, true)
}