From 0919ec67558c51b4f0bd271c1c21cd4944b682e4 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 6 Dec 2022 23:45:58 +0000 Subject: [PATCH] Ensure cluster-signing CA files contain only a single CA cert Signed-off-by: Brad Davidson --- pkg/daemons/config/types.go | 4 ++ pkg/daemons/control/deps/deps.go | 65 +++++++++++++++++++++++++++++--- pkg/daemons/control/server.go | 12 +++--- 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 1f67757885..2b9047e902 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -286,6 +286,10 @@ type ControlRuntime struct { ClientKubeAPIKey string NodePasswdFile string + SigningClientCA string + SigningServerCA string + ServiceCurrentKey string + KubeConfigAdmin string KubeConfigController string KubeConfigScheduler string diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index 352c1191d7..e439b8c7c7 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "net" "os" "path/filepath" @@ -30,6 +31,7 @@ import ( "k8s.io/apiserver/pkg/apis/apiserver" apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/client-go/util/keyutil" ) const ( @@ -110,6 +112,10 @@ func CreateRuntimeCertFiles(config *config.Control) { runtime.PasswdFile = filepath.Join(config.DataDir, "cred", "passwd") runtime.NodePasswdFile = filepath.Join(config.DataDir, "cred", "node-passwd") + runtime.SigningClientCA = filepath.Join(config.DataDir, "tls", "client-ca.nochain.crt") + runtime.SigningServerCA = filepath.Join(config.DataDir, "tls", "server-ca.nochain.crt") + runtime.ServiceCurrentKey = filepath.Join(config.DataDir, "tls", "service.current.key") + runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig") runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig") runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig") @@ -315,6 +321,18 @@ func genClientCerts(config *config.Control) error { return err } + certs, err := certutil.CertsFromFile(runtime.ClientCA) + if err != nil { + return err + } + + // If our CA certs are signed by a root or intermediate CA, ClientCA will contain a chain. + // The controller-manager's signer wants just a single cert, not a full chain; so create a file + // that is guaranteed to contain only a single certificate. + if err := certutil.WriteCert(runtime.SigningClientCA, certutil.EncodeCertPEM(certs[0])); err != nil { + return err + } + factory := getSigningCertFactory(regen, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.ClientCA, runtime.ClientCAKey) var certGen bool @@ -486,7 +504,24 @@ func createServerSigningCertKey(config *config.Control) (bool, error) { } return true, nil } - return createSigningCertKey(version.Program+"-server", runtime.ServerCA, runtime.ServerCAKey) + regen, err := createSigningCertKey(version.Program+"-server", runtime.ServerCA, runtime.ServerCAKey) + if err != nil { + return regen, err + } + + // If our CA certs are signed by a root or intermediate CA, ServerCA will contain a chain. + // The controller-manager's signer wants just a single cert, not a full chain; so create a file + // that is guaranteed to contain only a single certificate. + certs, err := certutil.CertsFromFile(runtime.ServerCA) + if err != nil { + return regen, err + } + + if err := certutil.WriteCert(runtime.SigningServerCA, certutil.EncodeCertPEM(certs[0])); err != nil { + return regen, err + } + + return regen, nil } func addSANs(altNames *certutil.AltNames, sans []string) { @@ -612,17 +647,35 @@ func exists(files ...string) bool { } func genServiceAccount(runtime *config.ControlRuntime) error { - _, keyErr := os.Stat(runtime.ServiceKey) - if keyErr == nil { - return nil + if _, err := os.Stat(runtime.ServiceKey); err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return err + } + key, err := certutil.NewPrivateKey() + if err != nil { + return err + } + if err := certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key)); err != nil { + return err + } } - key, err := certutil.NewPrivateKey() + // When rotating the ServiceAccount signing key, it is necessary to keep the old keys in ServiceKey so that + // old ServiceAccount tokens can be validated during the switchover process. The first key in the file + // should be the current key used to sign ServiceAccount tokens; others are old keys used for verification + // only. Create a file containing just the first key in the list, which will be used to configure the + // signing controller. + key, err := keyutil.PrivateKeyFromFile(runtime.ServiceKey) if err != nil { return err } - return certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key)) + keyData, err := keyutil.MarshalPrivateKeyToPEM(key) + if err != nil { + return err + } + + return certutil.WriteKey(runtime.ServiceCurrentKey, keyData) } func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) { diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 0104cfa02c..a4e6b52d39 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -94,7 +94,7 @@ func controllerManager(ctx context.Context, cfg *config.Control) error { "kubeconfig": runtime.KubeConfigController, "authorization-kubeconfig": runtime.KubeConfigController, "authentication-kubeconfig": runtime.KubeConfigController, - "service-account-private-key-file": runtime.ServiceKey, + "service-account-private-key-file": runtime.ServiceCurrentKey, "allocate-node-cidrs": "true", "service-cluster-ip-range": util.JoinIPNets(cfg.ServiceIPRanges), "cluster-cidr": util.JoinIPNets(cfg.ClusterIPRanges), @@ -103,13 +103,13 @@ func controllerManager(ctx context.Context, cfg *config.Control) error { "bind-address": cfg.Loopback(false), "secure-port": "10257", "use-service-account-credentials": "true", - "cluster-signing-kube-apiserver-client-cert-file": runtime.ClientCA, + "cluster-signing-kube-apiserver-client-cert-file": runtime.SigningClientCA, "cluster-signing-kube-apiserver-client-key-file": runtime.ClientCAKey, - "cluster-signing-kubelet-client-cert-file": runtime.ClientCA, + "cluster-signing-kubelet-client-cert-file": runtime.SigningClientCA, "cluster-signing-kubelet-client-key-file": runtime.ClientCAKey, - "cluster-signing-kubelet-serving-cert-file": runtime.ServerCA, + "cluster-signing-kubelet-serving-cert-file": runtime.SigningServerCA, "cluster-signing-kubelet-serving-key-file": runtime.ServerCAKey, - "cluster-signing-legacy-unknown-cert-file": runtime.ServerCA, + "cluster-signing-legacy-unknown-cert-file": runtime.SigningServerCA, "cluster-signing-legacy-unknown-key-file": runtime.ServerCAKey, } if cfg.NoLeaderElect { @@ -159,7 +159,7 @@ func apiServer(ctx context.Context, cfg *config.Control) error { argsMap["cert-dir"] = certDir argsMap["allow-privileged"] = "true" argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",") - argsMap["service-account-signing-key-file"] = runtime.ServiceKey + argsMap["service-account-signing-key-file"] = runtime.ServiceCurrentKey argsMap["service-cluster-ip-range"] = util.JoinIPNets(cfg.ServiceIPRanges) argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String() argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)