mirror of
https://github.com/k3s-io/k3s.git
synced 2024-06-07 19:41:36 +00:00
482 lines
15 KiB
Go
482 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math/big"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/blang/semver/v4"
|
|
"github.com/k3s-io/k3s/pkg/cluster"
|
|
"github.com/k3s-io/k3s/pkg/daemons/config"
|
|
"github.com/k3s-io/k3s/pkg/secretsencrypt"
|
|
"github.com/k3s-io/k3s/pkg/util"
|
|
"github.com/rancher/wrangler/pkg/generated/controllers/core"
|
|
"github.com/sirupsen/logrus"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
|
|
"k8s.io/client-go/util/retry"
|
|
"k8s.io/utils/ptr"
|
|
)
|
|
|
|
const aescbcKeySize = 32
|
|
|
|
type EncryptionState struct {
|
|
Stage string `json:"stage"`
|
|
ActiveKey string `json:"activekey"`
|
|
Enable *bool `json:"enable,omitempty"`
|
|
HashMatch bool `json:"hashmatch,omitempty"`
|
|
HashError string `json:"hasherror,omitempty"`
|
|
InactiveKeys []string `json:"inactivekeys,omitempty"`
|
|
}
|
|
|
|
type EncryptionRequest struct {
|
|
Stage *string `json:"stage,omitempty"`
|
|
Enable *bool `json:"enable,omitempty"`
|
|
Force bool `json:"force"`
|
|
Skip bool `json:"skip"`
|
|
}
|
|
|
|
func getEncryptionRequest(req *http.Request) (EncryptionRequest, error) {
|
|
b, err := io.ReadAll(req.Body)
|
|
if err != nil {
|
|
return EncryptionRequest{}, err
|
|
}
|
|
result := EncryptionRequest{}
|
|
err = json.Unmarshal(b, &result)
|
|
return result, err
|
|
}
|
|
|
|
func encryptionStatusHandler(server *config.Control) http.Handler {
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
if req.TLS == nil {
|
|
resp.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
status, err := encryptionStatus(server)
|
|
if err != nil {
|
|
genErrorMessage(resp, http.StatusInternalServerError, err, "secrets-encrypt")
|
|
return
|
|
}
|
|
b, err := json.Marshal(status)
|
|
if err != nil {
|
|
genErrorMessage(resp, http.StatusInternalServerError, err, "secrets-encrypt")
|
|
return
|
|
}
|
|
resp.Write(b)
|
|
})
|
|
}
|
|
|
|
func encryptionStatus(server *config.Control) (EncryptionState, error) {
|
|
state := EncryptionState{}
|
|
providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime)
|
|
if os.IsNotExist(err) {
|
|
return state, nil
|
|
} else if err != nil {
|
|
return state, err
|
|
}
|
|
if providers[1].Identity != nil && providers[0].AESCBC != nil {
|
|
state.Enable = ptr.To(true)
|
|
} else if providers[0].Identity != nil && providers[1].AESCBC != nil || !server.EncryptSecrets {
|
|
state.Enable = ptr.To(false)
|
|
}
|
|
|
|
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), ""); err != nil {
|
|
state.HashMatch = false
|
|
state.HashError = err.Error()
|
|
} else {
|
|
state.HashMatch = true
|
|
}
|
|
stage, _, err := getEncryptionHashAnnotation(server.Runtime.Core.Core())
|
|
if err != nil {
|
|
return state, err
|
|
}
|
|
state.Stage = stage
|
|
active := true
|
|
for _, p := range providers {
|
|
if p.AESCBC != nil {
|
|
for _, aesKey := range p.AESCBC.Keys {
|
|
if active {
|
|
active = false
|
|
state.ActiveKey = aesKey.Name
|
|
} else {
|
|
state.InactiveKeys = append(state.InactiveKeys, aesKey.Name)
|
|
}
|
|
}
|
|
}
|
|
if p.Identity != nil {
|
|
active = false
|
|
}
|
|
}
|
|
|
|
return state, nil
|
|
}
|
|
|
|
func encryptionEnable(ctx context.Context, server *config.Control, enable bool) error {
|
|
providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(providers) > 2 {
|
|
return fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
|
|
}
|
|
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if providers[1].Identity != nil && providers[0].AESCBC != nil && !enable {
|
|
logrus.Infoln("Disabling secrets encryption")
|
|
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
|
|
return err
|
|
}
|
|
} else if !enable {
|
|
logrus.Infoln("Secrets encryption already disabled")
|
|
return nil
|
|
} else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable {
|
|
logrus.Infoln("Enabling secrets encryption")
|
|
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
|
|
return err
|
|
}
|
|
} else if enable {
|
|
logrus.Infoln("Secrets encryption already enabled")
|
|
return nil
|
|
} else {
|
|
return fmt.Errorf("unable to enable/disable secrets encryption, unknown configuration")
|
|
}
|
|
if err := cluster.Save(ctx, server, true); err != nil {
|
|
return err
|
|
}
|
|
server.EncryptSkip = true
|
|
return setReencryptAnnotation(server)
|
|
}
|
|
|
|
func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler {
|
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
|
if req.TLS == nil {
|
|
resp.WriteHeader(http.StatusNotFound)
|
|
return
|
|
}
|
|
if req.Method != http.MethodPut {
|
|
resp.WriteHeader(http.StatusBadRequest)
|
|
return
|
|
}
|
|
encryptReq, err := getEncryptionRequest(req)
|
|
if err != nil {
|
|
resp.WriteHeader(http.StatusBadRequest)
|
|
resp.Write([]byte(err.Error()))
|
|
return
|
|
}
|
|
if encryptReq.Stage != nil {
|
|
switch *encryptReq.Stage {
|
|
case secretsencrypt.EncryptionPrepare:
|
|
err = encryptionPrepare(ctx, server, encryptReq.Force)
|
|
case secretsencrypt.EncryptionRotate:
|
|
err = encryptionRotate(ctx, server, encryptReq.Force)
|
|
case secretsencrypt.EncryptionRotateKeys:
|
|
err = encryptionRotateKeys(ctx, server)
|
|
case secretsencrypt.EncryptionReencryptActive:
|
|
err = encryptionReencrypt(ctx, server, encryptReq.Force, encryptReq.Skip)
|
|
default:
|
|
err = fmt.Errorf("unknown stage %s requested", *encryptReq.Stage)
|
|
}
|
|
} else if encryptReq.Enable != nil {
|
|
err = encryptionEnable(ctx, server, *encryptReq.Enable)
|
|
}
|
|
|
|
if err != nil {
|
|
genErrorMessage(resp, http.StatusBadRequest, err, "secrets-encrypt")
|
|
return
|
|
}
|
|
// If a user kills the k3s server immediately after this call, we run into issues where the files
|
|
// have not yet been written. This sleep ensures that things have time to sync to disk before
|
|
// the request completes.
|
|
time.Sleep(1 * time.Second)
|
|
resp.WriteHeader(http.StatusOK)
|
|
})
|
|
}
|
|
|
|
func encryptionPrepare(ctx context.Context, server *config.Control, force bool) error {
|
|
states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished
|
|
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil && !force {
|
|
return err
|
|
}
|
|
|
|
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := AppendNewEncryptionKey(&curKeys); err != nil {
|
|
return err
|
|
}
|
|
logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1])
|
|
|
|
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil {
|
|
return err
|
|
}
|
|
nodeName := os.Getenv("NODE_NAME")
|
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return cluster.Save(ctx, server, true)
|
|
}
|
|
|
|
func encryptionRotate(ctx context.Context, server *config.Control, force bool) error {
|
|
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionPrepare); err != nil && !force {
|
|
return err
|
|
}
|
|
|
|
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Right rotate elements
|
|
rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...)
|
|
|
|
if err = secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true); err != nil {
|
|
return err
|
|
}
|
|
logrus.Infoln("Encryption keys right rotated")
|
|
nodeName := os.Getenv("NODE_NAME")
|
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return cluster.Save(ctx, server, true)
|
|
}
|
|
|
|
func encryptionReencrypt(ctx context.Context, server *config.Control, force bool, skip bool) error {
|
|
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionRotate); err != nil && !force {
|
|
return err
|
|
}
|
|
server.EncryptForce = force
|
|
server.EncryptSkip = skip
|
|
nodeName := os.Getenv("NODE_NAME")
|
|
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash
|
|
node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann
|
|
if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil {
|
|
return err
|
|
}
|
|
logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
|
|
return nil
|
|
}
|
|
|
|
func addAndRotateKeys(server *config.Control) error {
|
|
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := AppendNewEncryptionKey(&curKeys); err != nil {
|
|
return err
|
|
}
|
|
logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1])
|
|
|
|
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Right rotate elements
|
|
rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...)
|
|
logrus.Infoln("Rotating secrets-encryption keys")
|
|
return secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true)
|
|
}
|
|
|
|
// encryptionRotateKeys is both adds and rotates keys, and sets the annotaiton that triggers the
|
|
// reencryption process. It is the preferred way to rotate keys, starting with v1.28
|
|
func encryptionRotateKeys(ctx context.Context, server *config.Control) error {
|
|
states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished
|
|
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := verifyRotateKeysSupport(server.Runtime.Core.Core()); err != nil {
|
|
return err
|
|
}
|
|
|
|
reloadTime, reloadSuccesses, err := secretsencrypt.GetEncryptionConfigMetrics(server.Runtime, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := addAndRotateKeys(server); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := secretsencrypt.WaitForEncryptionConfigReload(server.Runtime, reloadSuccesses, reloadTime); err != nil {
|
|
return err
|
|
}
|
|
|
|
return setReencryptAnnotation(server)
|
|
}
|
|
|
|
func setReencryptAnnotation(server *config.Control) error {
|
|
nodeName := os.Getenv("NODE_NAME")
|
|
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash
|
|
node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann
|
|
if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil {
|
|
return err
|
|
}
|
|
logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
|
|
return nil
|
|
}
|
|
|
|
func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error {
|
|
aescbcKey := make([]byte, aescbcKeySize)
|
|
_, err := rand.Read(aescbcKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
encodedKey := base64.StdEncoding.EncodeToString(aescbcKey)
|
|
|
|
newKey := []apiserverconfigv1.Key{
|
|
{
|
|
Name: "aescbckey-" + time.Now().Format(time.RFC3339),
|
|
Secret: encodedKey,
|
|
},
|
|
}
|
|
*keys = append(*keys, newKey...)
|
|
return nil
|
|
}
|
|
|
|
func getEncryptionHashAnnotation(core core.Interface) (string, string, error) {
|
|
nodeName := os.Getenv("NODE_NAME")
|
|
node, err := core.V1().Node().Get(nodeName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
if _, ok := node.Labels[util.ControlPlaneRoleLabelKey]; !ok {
|
|
return "", "", fmt.Errorf("cannot manage secrets encryption on non control-plane node %s", nodeName)
|
|
}
|
|
if ann, ok := node.Annotations[secretsencrypt.EncryptionHashAnnotation]; ok {
|
|
split := strings.Split(ann, "-")
|
|
if len(split) != 2 {
|
|
return "", "", fmt.Errorf("invalid annotation %s found on node %s", ann, nodeName)
|
|
}
|
|
return split[0], split[1], nil
|
|
}
|
|
return "", "", fmt.Errorf("missing annotation on node %s", nodeName)
|
|
}
|
|
|
|
// verifyRotateKeysSupport checks that the k3s version is at least v1.28.0 on all control-plane nodes
|
|
func verifyRotateKeysSupport(core core.Interface) error {
|
|
labelSelector := labels.Set{util.ControlPlaneRoleLabelKey: "true"}.String()
|
|
nodes, err := core.V1().Node().List(metav1.ListOptions{LabelSelector: labelSelector})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, node := range nodes.Items {
|
|
kubver, err := semver.ParseTolerant(node.Status.NodeInfo.KubeletVersion)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to parse kubelet version %s: %v", node.Status.NodeInfo.KubeletVersion, err)
|
|
}
|
|
supportVer, err := semver.Make("1.28.0")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if kubver.LT(supportVer) {
|
|
return fmt.Errorf("node %s is running k3s version %s that does not support rotate-keys", node.ObjectMeta.Name, kubver.String())
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// verifyEncryptionHashAnnotation checks that all nodes are on the same stage,
|
|
// and that a request for new stage is valid
|
|
func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.Interface, prevStage string) error {
|
|
var firstHash string
|
|
var firstNodeName string
|
|
first := true
|
|
labelSelector := labels.Set{util.ControlPlaneRoleLabelKey: "true"}.String()
|
|
nodes, err := core.V1().Node().List(metav1.ListOptions{LabelSelector: labelSelector})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, node := range nodes.Items {
|
|
hash, ok := node.Annotations[secretsencrypt.EncryptionHashAnnotation]
|
|
if ok && first {
|
|
firstHash = hash
|
|
first = false
|
|
firstNodeName = node.ObjectMeta.Name
|
|
} else if ok && hash != firstHash {
|
|
return fmt.Errorf("hash does not match between %s and %s", firstNodeName, node.ObjectMeta.Name)
|
|
}
|
|
}
|
|
|
|
if prevStage == "" {
|
|
return nil
|
|
}
|
|
|
|
oldStage, oldHash, err := getEncryptionHashAnnotation(core)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
encryptionConfigHash, err := secretsencrypt.GenEncryptionConfigHash(runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !strings.Contains(prevStage, oldStage) {
|
|
return fmt.Errorf("incorrect stage: %s found on node %s", oldStage, nodes.Items[0].ObjectMeta.Name)
|
|
} else if oldHash != encryptionConfigHash {
|
|
return fmt.Errorf("invalid hash: %s found on node %s", oldHash, nodes.Items[0].ObjectMeta.Name)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// genErrorMessage sends and logs a random error ID so that logs can be correlated
|
|
// between the REST API (which does not provide any detailed error output, to avoid
|
|
// information disclosure) and the server logs.
|
|
func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error, component string) {
|
|
errID, err := rand.Int(rand.Reader, big.NewInt(99999))
|
|
if err != nil {
|
|
resp.WriteHeader(http.StatusInternalServerError)
|
|
resp.Write([]byte(err.Error()))
|
|
return
|
|
}
|
|
logrus.Warnf("%s error ID %05d: %s", component, errID, passedErr.Error())
|
|
resp.WriteHeader(statusCode)
|
|
resp.Write([]byte(fmt.Sprintf("%s error ID %05d", component, errID)))
|
|
}
|