Add certificate rotate-ca to write updated CA certs to datastore

This command must be run on a server while the service is running. After this command completes, all the servers in the cluster should be restarted to load the new CA files.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

cmd/cert/main.go

@@ -17,7 +17,9 @@ func main() {
 	app.Commands = []cli.Command{
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
-				cert.Run),
+				cert.Rotate,
+				cert.RotateCA,
+			),
 		),
 	}
 

cmd/k3s/main.go

@@ -68,7 +68,9 @@ func main() {
 		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
-				certCommand),
+				certCommand,
+				certCommand,
+			),
 		),
 		cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)),
 	}

cmd/server/main.go

@@ -65,7 +65,9 @@ func main() {
 		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
-				cert.Run),
+				cert.Rotate,
+				cert.RotateCA,
+			),
 		),
 		cmds.NewCompletionCommand(completion.Run),
 	}

main.go

@@ -49,7 +49,9 @@ func main() {
 		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
-				cert.Run),
+				cert.Rotate,
+				cert.RotateCA,
+			),
 		),
 		cmds.NewCompletionCommand(completion.Run),
 	}

pkg/cli/cert/cert.go

@@ -1,20 +1,24 @@
 package cert
 
 import (
-	"errors"
+	"bytes"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
 	"time"
 
 	"github.com/erikdubbelboer/gspt"
+	"github.com/k3s-io/k3s/pkg/bootstrap"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
+	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
 	"github.com/k3s-io/k3s/pkg/datadir"
 	"github.com/k3s-io/k3s/pkg/server"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/otiai10/copy"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -47,19 +51,31 @@ var services = []string{
 	version.Program + k3sServerService,
 }
 
-func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string, string, error) {
+func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string, error) {
 	gspt.SetProcTitle(os.Args[0])
 
-	sc.ControlConfig.DataDir = cfg.DataDir
-	sc.ControlConfig.Runtime = &config.ControlRuntime{}
 	dataDir, err := datadir.Resolve(cfg.DataDir)
 	if err != nil {
-		return "", "", err
+		return "", err
 	}
-	return filepath.Join(dataDir, "server"), filepath.Join(dataDir, "agent"), err
+	sc.ControlConfig.DataDir = filepath.Join(dataDir, "server")
+
+	if cfg.Token == "" {
+		fp := filepath.Join(sc.ControlConfig.DataDir, "token")
+		tokenByte, err := os.ReadFile(fp)
+		if err != nil {
+			return "", err
+		}
+		cfg.Token = string(bytes.TrimRight(tokenByte, "\n"))
+	}
+	sc.ControlConfig.Token = cfg.Token
+
+	sc.ControlConfig.Runtime = &config.ControlRuntime{}
+
+	return dataDir, nil
 }
 
-func Run(app *cli.Context) error {
+func Rotate(app *cli.Context) error {
 	if err := cmds.InitLogging(); err != nil {
 		return err
 	}
@@ -69,27 +85,26 @@ func Run(app *cli.Context) error {
 func rotate(app *cli.Context, cfg *cmds.Server) error {
 	var serverConfig server.Config
 
-	serverDataDir, agentDataDir, err := commandSetup(app, cfg, &serverConfig)
+	dataDir, err := commandSetup(app, cfg, &serverConfig)
 	if err != nil {
 		return err
 	}
 
-	serverConfig.ControlConfig.DataDir = serverDataDir
-	serverConfig.ControlConfig.Runtime = &config.ControlRuntime{}
 	deps.CreateRuntimeCertFiles(&serverConfig.ControlConfig)
 
 	if err := validateCertConfig(); err != nil {
 		return err
 	}
 
-	tlsBackupDir, err := backupCertificates(serverDataDir, agentDataDir)
+	agentDataDir := filepath.Join(dataDir, "agent")
+	tlsBackupDir, err := backupCertificates(serverConfig.ControlConfig.DataDir, agentDataDir)
 	if err != nil {
 		return err
 	}
 
 	if len(cmds.ServicesList) == 0 {
-		// detecting if the service is an agent or server
-		_, err := os.Stat(serverDataDir)
+		// detecting if the command is being run on an agent or server
+		_, err := os.Stat(serverConfig.ControlConfig.DataDir)
 		if err != nil {
 			if !os.IsNotExist(err) {
 				return err
@@ -152,7 +167,7 @@ func rotate(app *cli.Context, cfg *cmds.Server) error {
 				serverConfig.ControlConfig.Runtime.ClientCloudControllerCert,
 				serverConfig.ControlConfig.Runtime.ClientCloudControllerKey)
 		case version.Program + k3sServerService:
-			dynamicListenerRegenFilePath := filepath.Join(serverDataDir, "tls", "dynamic-cert-regenerate")
+			dynamicListenerRegenFilePath := filepath.Join(serverConfig.ControlConfig.DataDir, "tls", "dynamic-cert-regenerate")
 			if err := os.WriteFile(dynamicListenerRegenFilePath, []byte{}, 0600); err != nil {
 				return err
 			}
@@ -254,3 +269,48 @@ func validateCertConfig() error {
 	}
 	return nil
 }
+
+func RotateCA(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	return rotateCA(app, &cmds.ServerConfig, &cmds.CertRotateCAConfig)
+}
+
+func rotateCA(app *cli.Context, cfg *cmds.Server, sync *cmds.CertRotateCA) error {
+	var serverConfig server.Config
+
+	_, err := commandSetup(app, cfg, &serverConfig)
+	if err != nil {
+		return err
+	}
+
+	info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, serverConfig.ControlConfig.Token, "server")
+	if err != nil {
+		return err
+	}
+
+	// Set up dummy server config for reading new bootstrap data from disk.
+	tmpServer := &config.Control{
+		Runtime: &config.ControlRuntime{},
+		DataDir: filepath.Dir(sync.CACertPath),
+	}
+	deps.CreateRuntimeCertFiles(tmpServer)
+
+	// Override these paths so that we don't get warnings when they don't exist, as the user is not expected to provide them.
+	tmpServer.Runtime.PasswdFile = "/dev/null"
+	tmpServer.Runtime.IPSECKey = "/dev/null"
+
+	buf := &bytes.Buffer{}
+	if err := bootstrap.ReadFromDisk(buf, &tmpServer.Runtime.ControlRuntimeBootstrap); err != nil {
+		return err
+	}
+
+	url := fmt.Sprintf("/v1-%s/cert/cacerts?force=%t", version.Program, sync.Force)
+	if err = info.Put(url, buf.Bytes()); err != nil {
+		return errors.Wrap(err, "see server log for details")
+	}
+
+	fmt.Println("certificates saved to datastore")
+	return nil
+}

pkg/cli/cmds/certs.go

@@ -7,9 +7,15 @@ import (
 
 const CertCommand = "certificate"
 
+type CertRotateCA struct {
+	CACertPath string
+	Force      bool
+}
+
 var (
-	ServicesList     cli.StringSlice
-	CertCommandFlags = []cli.Flag{
+	ServicesList           cli.StringSlice
+	CertRotateCAConfig     CertRotateCA
+	CertRotateCommandFlags = []cli.Flag{
 		DebugFlag,
 		ConfigFlag,
 		LogFile,
@@ -21,28 +27,55 @@ var (
 			Value: &ServicesList,
 		},
 	}
+	CertRotateCACommandFlags = []cli.Flag{
+		cli.StringFlag{
+			Name:        "server,s",
+			Usage:       "(cluster) Server to connect to",
+			EnvVar:      version.ProgramUpper + "_URL",
+			Value:       "https://127.0.0.1:6443",
+			Destination: &ServerConfig.ServerURL,
+		},
+		cli.StringFlag{
+			Name:        "path",
+			Usage:       "Path to directory containing new CA certificates",
+			Destination: &CertRotateCAConfig.CACertPath,
+			Required:    true,
+		},
+		cli.BoolFlag{
+			Name:        "force",
+			Usage:       "Force certificate replacement, even if consistency checks fail",
+			Destination: &CertRotateCAConfig.Force,
+		},
+	}
 )
 
 func NewCertCommand(subcommands []cli.Command) cli.Command {
 	return cli.Command{
 		Name:            CertCommand,
-		Usage:           "Certificates management",
+		Usage:           "Manage K3s certificates",
 		SkipFlagParsing: false,
 		SkipArgReorder:  true,
 		Subcommands:     subcommands,
-		Flags:           CertCommandFlags,
 	}
 }
 
-func NewCertSubcommands(rotate func(ctx *cli.Context) error) []cli.Command {
+func NewCertSubcommands(rotate, rotateCA func(ctx *cli.Context) error) []cli.Command {
 	return []cli.Command{
 		{
 			Name:            "rotate",
-			Usage:           "Certificate rotation",
+			Usage:           "Rotate " + version.Program + " component certificates on disk",
 			SkipFlagParsing: false,
 			SkipArgReorder:  true,
 			Action:          rotate,
-			Flags:           CertCommandFlags,
+			Flags:           CertRotateCommandFlags,
+		},
+		{
+			Name:            "rotate-ca",
+			Usage:           "Write updated " + version.Program + " CA certificates to the datastore",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          rotateCA,
+			Flags:           CertRotateCACommandFlags,
 		},
 	}
 }

pkg/clientaccess/token.go

@@ -217,8 +217,13 @@ func (i *Info) Get(path string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	u.Path = path
-	return get(u.String(), GetHTTPClient(i.CACerts), i.Username, i.Password)
+	p, err := url.Parse(path)
+	if err != nil {
+		return nil, err
+	}
+	p.Scheme = u.Scheme
+	p.Host = u.Host
+	return get(p.String(), GetHTTPClient(i.CACerts), i.Username, i.Password)
 }
 
 // Put makes a request to a subpath of info's BaseURL
@@ -227,8 +232,13 @@ func (i *Info) Put(path string, body []byte) error {
 	if err != nil {
 		return err
 	}
-	u.Path = path
-	return put(u.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password)
+	p, err := url.Parse(path)
+	if err != nil {
+		return err
+	}
+	p.Scheme = u.Scheme
+	p.Host = u.Host
+	return put(p.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password)
 }
 
 // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
@@ -326,7 +336,7 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		return nil, fmt.Errorf("%s: %s", u, resp.Status)
 	}
 
@@ -352,7 +362,7 @@ func put(u string, body []byte, client *http.Client, username, password string)
 	defer resp.Body.Close()
 
 	respBody, _ := io.ReadAll(resp.Body)
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		return fmt.Errorf("%s: %s %s", u, resp.Status, string(respBody))
 	}
 

pkg/server/cert.go

@@ -0,0 +1,206 @@
+package server
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/k3s-io/k3s/pkg/bootstrap"
+	"github.com/k3s-io/k3s/pkg/cluster"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/pkg/errors"
+	certutil "github.com/rancher/dynamiclistener/cert"
+	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/util/keyutil"
+)
+
+func caCertReplaceHandler(server *config.Control) http.HandlerFunc {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil || req.Method != http.MethodPut {
+			resp.WriteHeader(http.StatusNotFound)
+			return
+		}
+		force, _ := strconv.ParseBool(req.FormValue("force"))
+		if err := caCertReplace(server, req.Body, force); err != nil {
+			genErrorMessage(resp, http.StatusInternalServerError, err, "certificate")
+			return
+		}
+		logrus.Infof("certificate: Cluster Certificate Authority data has been updated, %s must be restarted.", version.Program)
+		resp.WriteHeader(http.StatusNoContent)
+	})
+}
+
+// caCertReplace stores new CA Certificate data from the client.  The data is temporarily written out to disk,
+// validated to confirm that the new certs share a common root with the existing certs, and if so are saved to
+// the datastore.  If the functions succeeds, servers should be restarted immediately to load the new certs
+// from the bootstrap data.
+func caCertReplace(server *config.Control, buf io.ReadCloser, force bool) error {
+	tmpdir, err := os.MkdirTemp("", "cacerts")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpdir)
+
+	tmpServer := &config.Control{
+		Runtime: &config.ControlRuntime{
+			EtcdConfig:  server.Runtime.EtcdConfig,
+			ServerToken: server.Runtime.ServerToken,
+		},
+		Token:   server.Token,
+		DataDir: tmpdir,
+	}
+	deps.CreateRuntimeCertFiles(tmpServer)
+
+	bootstrapData := bootstrap.PathsDataformat{}
+	if err := json.NewDecoder(buf).Decode(&bootstrapData); err != nil {
+		return err
+	}
+
+	if err := bootstrap.WriteToDiskFromStorage(bootstrapData, &tmpServer.Runtime.ControlRuntimeBootstrap); err != nil {
+		return err
+	}
+
+	if err := validateBootstrap(server, tmpServer); err != nil {
+		if !force {
+			return errors.Wrap(err, "failed to validate new CA certificates and keys")
+		}
+		logrus.Warnf("Save of CA certificates and keys forced, ignoring validation errors: %v", err)
+	}
+
+	return cluster.Save(context.TODO(), tmpServer, true)
+}
+
+// validateBootstrap checks the new certs and keys to ensure that the cluster would function properly were they to be used.
+// - The new leaf CA certificates must be verifiable using the same root and intermediate certs as the current leaf CA certificates.
+// - The new service account signing key bundle must include the currently active signing key.
+func validateBootstrap(oldServer, newServer *config.Control) error {
+	errs := []error{}
+
+	// Use reflection to iterate over all of the bootstrap fields, checking files at each of the new paths.
+	oldMeta := reflect.ValueOf(&oldServer.Runtime.ControlRuntimeBootstrap).Elem()
+	newMeta := reflect.ValueOf(&newServer.Runtime.ControlRuntimeBootstrap).Elem()
+	for _, field := range reflect.VisibleFields(oldMeta.Type()) {
+		oldVal := oldMeta.FieldByName(field.Name)
+		newVal := newMeta.FieldByName(field.Name)
+
+		info, err := os.Stat(newVal.String())
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			errs = append(errs, errors.Wrap(err, field.Name))
+			continue
+		}
+
+		if info == nil || info.Size() == 0 {
+			if newVal.CanSet() {
+				logrus.Infof("certificate: %s not provided; using current value", field.Name)
+				newVal.Set(oldVal)
+			} else {
+				errs = append(errs, fmt.Errorf("cannot use current data for %s; field is not settable", field.Name))
+			}
+		}
+
+		// Check CA chain consistency and cert/key agreement
+		if strings.HasSuffix(field.Name, "CA") {
+			if err := validateCA(oldVal.String(), newVal.String()); err != nil {
+				errs = append(errs, errors.Wrap(err, field.Name))
+			}
+			newKeyVal := newMeta.FieldByName(field.Name + "Key")
+			if err := validateCAKey(newVal.String(), newKeyVal.String()); err != nil {
+				errs = append(errs, errors.Wrap(err, field.Name+"Key"))
+			}
+		}
+
+		// Check signing key rotation
+		if field.Name == "ServiceKey" {
+			if err := validateServiceKey(oldVal.String(), newVal.String()); err != nil {
+				errs = append(errs, errors.Wrap(err, field.Name))
+			}
+		}
+	}
+
+	if len(errs) > 0 {
+		return merr.NewErrors(errs...)
+	}
+	return nil
+}
+
+func validateCA(oldCAPath, newCAPath string) error {
+	oldCerts, err := certutil.CertsFromFile(oldCAPath)
+	if err != nil {
+		return err
+	}
+
+	if len(oldCerts) == 1 {
+		return errors.New("old CA is self-signed")
+	}
+
+	newCerts, err := certutil.CertsFromFile(newCAPath)
+	if err != nil {
+		return err
+	}
+
+	if len(newCerts) == 1 {
+		return errors.New("new CA is self-signed")
+	}
+
+	roots := x509.NewCertPool()
+	intermediates := x509.NewCertPool()
+	for i, cert := range oldCerts {
+		if i > 0 {
+			if len(cert.AuthorityKeyId) == 0 || bytes.Equal(cert.AuthorityKeyId, cert.SubjectKeyId) {
+				roots.AddCert(cert)
+			} else {
+				intermediates.AddCert(cert)
+			}
+		}
+	}
+
+	_, err = newCerts[0].Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates})
+	if err != nil {
+		err = errors.Wrap(err, "new CA cert cannot be verified using old CA chain")
+	}
+	return err
+}
+
+// validateCAKey confirms that the private key is valid for the certificate
+func validateCAKey(newCAPath, newCAKeyPath string) error {
+	_, err := tls.LoadX509KeyPair(newCAPath, newCAKeyPath)
+	if err != nil {
+		err = errors.Wrap(err, "new CA cert and key cannot be loaded as X590KeyPair")
+	}
+	return err
+}
+
+// validateServiceKey ensures that the first key from the old serviceaccount signing key list
+// is also present in the new key list, to ensure that old signatures can still be validated.
+func validateServiceKey(oldKeyPath, newKeyPath string) error {
+	oldKeys, err := keyutil.PublicKeysFromFile(oldKeyPath)
+	if err != nil {
+		return err
+	}
+
+	newKeys, err := keyutil.PublicKeysFromFile(newKeyPath)
+	if err != nil {
+		return err
+	}
+
+	for _, key := range newKeys {
+		if reflect.DeepEqual(oldKeys[0], key) {
+			return nil
+		}
+	}
+
+	return errors.New("old ServiceAccount signing key not in new ServiceAccount key list")
+}

pkg/server/router.go

@@ -71,6 +71,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 	serverAuthed.Use(authMiddleware(serverConfig, version.Program+":server"))
 	serverAuthed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig))
 	serverAuthed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig))
+	serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
 	serverAuthed.Path("/db/info").Handler(nodeAuthed)
 	serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime))
 

pkg/server/secrets-encrypt.go

@@ -60,12 +60,12 @@ func encryptionStatusHandler(server *config.Control) http.Handler {
 		}
 		status, err := encryptionStatus(server)
 		if err != nil {
-			genErrorMessage(resp, http.StatusInternalServerError, err)
+			genErrorMessage(resp, http.StatusInternalServerError, err, "secrets-encrypt")
 			return
 		}
 		b, err := json.Marshal(status)
 		if err != nil {
-			genErrorMessage(resp, http.StatusInternalServerError, err)
+			genErrorMessage(resp, http.StatusInternalServerError, err, "secrets-encrypt")
 			return
 		}
 		resp.Write(b)
@@ -183,7 +183,7 @@ func encryptionConfigHandler(ctx context.Context, server *config.Control) http.H
 		}
 
 		if err != nil {
-			genErrorMessage(resp, http.StatusBadRequest, err)
+			genErrorMessage(resp, http.StatusBadRequest, err, "secrets-encrypt")
 			return
 		}
 		// If a user kills the k3s server immediately after this call, we run into issues where the files
@@ -364,14 +364,14 @@ func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.In
 // genErrorMessage sends and logs a random error ID so that logs can be correlated
 // between the REST API (which does not provide any detailed error output, to avoid
 // information disclosure) and the server logs.
-func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error) {
+func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error, component string) {
 	errID, err := rand.Int(rand.Reader, big.NewInt(99999))
 	if err != nil {
 		resp.WriteHeader(http.StatusInternalServerError)
 		resp.Write([]byte(err.Error()))
 		return
 	}
-	logrus.Warnf("secrets-encrypt error ID %05d: %s", errID, passedErr.Error())
+	logrus.Warnf("%s error ID %05d: %s", component, errID, passedErr.Error())
 	resp.WriteHeader(statusCode)
-	resp.Write([]byte(fmt.Sprintf("secrets-encrypt error ID %05d", errID)))
+	resp.Write([]byte(fmt.Sprintf("%s error ID %05d", component, errID)))
 }