Secrets-encryption rotation (#4372)

* Regular CLI framework for encrypt commands
* New secrets-encryption feature
* New integration test
* fixes for flaky integration test CI
* Fix to bootstrap on restart of existing nodes
* Consolidate event recorder

Signed-off-by: Derek Nola <derek.nola@suse.com>

.github/workflows/integration.yaml

@@ -56,17 +56,8 @@ jobs:
     - name: Run Integration Tests
       run: | 
         chmod +x ./dist/artifacts/k3s
-        go test -coverpkg=./... -coverprofile=coverage.out ./pkg/... -run Integration
+        go test ./pkg/... ./tests/integration/... -run Integration
-        go tool cover -func coverage.out
-        # these tests do not relate to coverage and must be run separately
-        go test ./tests/integration/... -run Integration 
     - name: On Failure, Launch Debug Session
       if: ${{ failure() }}
       uses: mxschmitt/action-tmate@v3
       timeout-minutes: 5
-    - name: Upload Results To Codecov
-      uses: codecov/codecov-action@v1
-      with:
-        files: ./coverage.out
-        flags: inttests # optional
-        verbose: true # optional (default = false)

cmd/encrypt/main.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
+	"github.com/rancher/k3s/pkg/configfilearg"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	app := cmds.NewApp()
+	app.Commands = []cli.Command{
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
+	}
+
+	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
+		logrus.Fatal(err)
+	}
+}

cmd/k3s/main.go

@@ -35,6 +35,7 @@ func main() {
 	}
 
 	etcdsnapshotCommand := internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)
+	secretsencryptCommand := internalCLIAction(version.Program+"-"+cmds.SecretsEncryptCommand, dataDir, os.Args)
 	certCommand := internalCLIAction(version.Program+"-"+cmds.CertCommand, dataDir, os.Args)
 
 	// Handle subcommand invocation (k3s server, k3s crictl, etc)
@@ -53,6 +54,15 @@ func main() {
 				etcdsnapshotCommand,
 				etcdsnapshotCommand),
 		),
+		cmds.NewSecretsEncryptCommand(secretsencryptCommand,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand,
+				secretsencryptCommand),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				certCommand),

cmd/server/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rancher/k3s/pkg/cli/ctr"
 	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/cli/kubectl"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
 	"github.com/rancher/k3s/pkg/cli/server"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/rancher/k3s/pkg/containerd"
@@ -53,6 +54,15 @@ func main() {
 				etcdsnapshot.Prune,
 				etcdsnapshot.Run),
 		),
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				cert.Run),

main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/rancher/k3s/pkg/cli/crictl"
 	"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
 	"github.com/rancher/k3s/pkg/cli/kubectl"
+	"github.com/rancher/k3s/pkg/cli/secretsencrypt"
 	"github.com/rancher/k3s/pkg/cli/server"
 	"github.com/rancher/k3s/pkg/configfilearg"
 	"github.com/sirupsen/logrus"
@@ -37,6 +38,15 @@ func main() {
 				etcdsnapshot.Prune,
 				etcdsnapshot.Run),
 		),
+		cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
+			cmds.NewSecretsEncryptSubcommands(
+				secretsencrypt.Status,
+				secretsencrypt.Enable,
+				secretsencrypt.Disable,
+				secretsencrypt.Prepare,
+				secretsencrypt.Rotate,
+				secretsencrypt.Reencrypt),
+		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(
 				cert.Run),

pkg/cli/cmds/agent.go

@@ -58,6 +58,12 @@ type AgentShared struct {
 var (
 	appName        = filepath.Base(os.Args[0])
 	AgentConfig    Agent
+	AgentTokenFlag = cli.StringFlag{
+		Name:        "token,t",
+		Usage:       "(cluster) Token to use for authentication",
+		EnvVar:      version.ProgramUpper + "_TOKEN",
+		Destination: &AgentConfig.Token,
+	}
 	NodeIPFlag = cli.StringSliceFlag{
 		Name:  "node-ip,i",
 		Usage: "(agent/networking) IPv4/IPv6 addresses to advertise for node",
@@ -217,12 +223,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			VModule,
 			LogFile,
 			AlsoLogToStderr,
-			cli.StringFlag{
+			AgentTokenFlag,
-				Name:        "token,t",
-				Usage:       "(cluster) Token to use for authentication",
-				EnvVar:      version.ProgramUpper + "_TOKEN",
-				Destination: &AgentConfig.Token,
-			},
 			cli.StringFlag{
 				Name:        "token-file",
 				Usage:       "(cluster) Token file to use for authentication",

pkg/cli/cmds/etcd_snapshot.go

@@ -20,11 +20,7 @@ var EtcdSnapshotFlags = []cli.Flag{
 		EnvVar:      version.ProgramUpper + "_NODE_NAME",
 		Destination: &AgentConfig.NodeName,
 	},
-	cli.StringFlag{
+	DataDirFlag,
-		Name:        "data-dir,d",
-		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
-		Destination: &ServerConfig.DataDir,
-	},
 	&cli.StringFlag{
 		Name:        "dir,etcd-snapshot-dir",
 		Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",

pkg/cli/cmds/secrets_encrypt.go

@@ -0,0 +1,94 @@
+package cmds
+
+import (
+	"github.com/urfave/cli"
+)
+
+const SecretsEncryptCommand = "secrets-encrypt"
+
+var EncryptFlags = []cli.Flag{
+	DataDirFlag,
+	ServerToken,
+}
+
+func NewSecretsEncryptCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command {
+	return cli.Command{
+		Name:            SecretsEncryptCommand,
+		Usage:           "Control secrets encryption and keys rotation",
+		SkipFlagParsing: false,
+		SkipArgReorder:  true,
+		Action:          action,
+		Subcommands:     subcommands,
+	}
+}
+
+func NewSecretsEncryptSubcommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) []cli.Command {
+	return []cli.Command{
+		{
+			Name:            "status",
+			Usage:           "Print current status of secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          status,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "enable",
+			Usage:           "Enable secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          enable,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "disable",
+			Usage:           "Disable secrets encryption",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          disable,
+			Flags:           EncryptFlags,
+		},
+		{
+			Name:            "prepare",
+			Usage:           "Prepare for encryption keys rotation",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          prepare,
+			Flags: append(EncryptFlags, &cli.BoolFlag{
+				Name:        "f,force",
+				Usage:       "Force preparation.",
+				Destination: &ServerConfig.EncryptForce,
+			}),
+		},
+		{
+			Name:            "rotate",
+			Usage:           "Rotate secrets encryption keys",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          rotate,
+			Flags: append(EncryptFlags, &cli.BoolFlag{
+				Name:        "f,force",
+				Usage:       "Force key rotation.",
+				Destination: &ServerConfig.EncryptForce,
+			}),
+		},
+		{
+			Name:            "reencrypt",
+			Usage:           "Reencrypt all data with new encryption key",
+			SkipFlagParsing: false,
+			SkipArgReorder:  true,
+			Action:          reencrypt,
+			Flags: append(EncryptFlags,
+				&cli.BoolFlag{
+					Name:        "f,force",
+					Usage:       "Force secrets reencryption.",
+					Destination: &ServerConfig.EncryptForce,
+				},
+				&cli.BoolFlag{
+					Name:        "skip",
+					Usage:       "Skip removing old key",
+					Destination: &ServerConfig.EncryptSkip,
+				}),
+		},
+	}
+}

pkg/cli/cmds/server.go

@@ -74,6 +74,8 @@ type Server struct {
 	ClusterReset             bool
 	ClusterResetRestorePath  string
 	EncryptSecrets           bool
+	EncryptForce             bool
+	EncryptSkip              bool
 	SystemDefaultRegistry    string
 	StartupHooks             []StartupHook
 	EtcdSnapshotName         string
@@ -97,6 +99,17 @@ type Server struct {
 
 var (
 	ServerConfig Server
+	DataDirFlag  = cli.StringFlag{
+		Name:        "data-dir,d",
+		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
+		Destination: &ServerConfig.DataDir,
+	}
+	ServerToken = cli.StringFlag{
+		Name:        "token,t",
+		Usage:       "(cluster) Shared secret used to join a server or agent to a cluster",
+		Destination: &ServerConfig.Token,
+		EnvVar:      version.ProgramUpper + "_TOKEN",
+	}
 	ClusterCIDR = cli.StringSliceFlag{
 		Name:  "cluster-cidr",
 		Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)",
@@ -179,11 +192,7 @@ var ServerFlags = []cli.Flag{
 		Usage: "(listener) Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert",
 		Value: &ServerConfig.TLSSan,
 	},
-	cli.StringFlag{
+	DataDirFlag,
-		Name:        "data-dir,d",
-		Usage:       "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
-		Destination: &ServerConfig.DataDir,
-	},
 	ClusterCIDR,
 	ServiceCIDR,
 	ServiceNodePortRange,
@@ -195,12 +204,7 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.FlannelBackend,
 		Value:       "vxlan",
 	},
-	cli.StringFlag{
+	ServerToken,
-		Name:        "token,t",
-		Usage:       "(cluster) Shared secret used to join a server or agent to a cluster",
-		Destination: &ServerConfig.Token,
-		EnvVar:      version.ProgramUpper + "_TOKEN",
-	},
 	cli.StringFlag{
 		Name:        "token-file",
 		Usage:       "(cluster) File containing the cluster-secret/token",

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -0,0 +1,216 @@
+package secretsencrypt
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"text/tabwriter"
+
+	"github.com/erikdubbelboer/gspt"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/secretsencrypt"
+	"github.com/rancher/k3s/pkg/server"
+	"github.com/rancher/k3s/pkg/version"
+	"github.com/urfave/cli"
+	"k8s.io/utils/pointer"
+)
+
+func commandPrep(app *cli.Context, cfg *cmds.Server) (config.Control, *clientaccess.Info, error) {
+	var controlConfig config.Control
+	var err error
+	// hide process arguments from ps output, since they may contain
+	// database credentials or other secrets.
+	gspt.SetProcTitle(os.Args[0] + " encrypt")
+
+	controlConfig.DataDir, err = server.ResolveDataDir(cfg.DataDir)
+	if err != nil {
+		return controlConfig, nil, err
+	}
+	if cfg.ServerURL == "" {
+		cfg.ServerURL = "https://127.0.0.1:6443"
+	}
+
+	if cfg.Token == "" {
+		fp := filepath.Join(controlConfig.DataDir, "token")
+		tokenByte, err := ioutil.ReadFile(fp)
+		if err != nil {
+			return controlConfig, nil, err
+		}
+		controlConfig.Token = string(bytes.TrimRight(tokenByte, "\n"))
+	} else {
+		controlConfig.Token = cfg.Token
+	}
+	controlConfig.EncryptForce = cfg.EncryptForce
+	controlConfig.EncryptSkip = cfg.EncryptSkip
+	info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, controlConfig.Token, "node")
+	if err != nil {
+		return controlConfig, nil, err
+	}
+	return controlConfig, info, nil
+}
+
+func Enable(app *cli.Context) error {
+	var err error
+	if err = cmds.InitLogging(); err != nil {
+		return err
+	}
+	_, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(true)})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("secrets-encryption enabled")
+	return nil
+}
+
+func Disable(app *cli.Context) error {
+
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	_, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(false)})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("secrets-encryption disabled")
+	return nil
+}
+
+func Status(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	_, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	data, err := info.Get("/v1-" + version.Program + "/encrypt/status")
+	if err != nil {
+		return err
+	}
+	status := server.EncryptionState{}
+	if err := json.Unmarshal(data, &status); err != nil {
+		return err
+	}
+
+	if status.Enable == nil {
+		fmt.Println("Encryption Status: Disabled, no configuration file found")
+		return nil
+	}
+
+	var statusOutput string
+	if *status.Enable {
+		statusOutput += "Encryption Status: Enabled\n"
+	} else {
+		statusOutput += "Encryption Status: Disabled\n"
+	}
+	statusOutput += fmt.Sprintln("Current Rotation Stage:", status.Stage)
+
+	if status.HashMatch {
+		statusOutput += fmt.Sprintln("Server Encryption Hashes: All hashes match")
+	} else {
+		statusOutput += fmt.Sprintf("Server Encryption Hashes: %s\n", status.HashError)
+	}
+
+	var tabBuffer bytes.Buffer
+	w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0)
+	fmt.Fprintf(w, "\n")
+	fmt.Fprintf(w, "Active\tKey Type\tName\n")
+	fmt.Fprintf(w, "------\t--------\t----\n")
+	if status.ActiveKey != "" {
+		fmt.Fprintf(w, " *\t%s\t%s\n", "AES-CBC", status.ActiveKey)
+	}
+	for _, k := range status.InactiveKeys {
+		fmt.Fprintf(w, "\t%s\t%s\n", "AES-CBC", k)
+	}
+	w.Flush()
+	fmt.Println(statusOutput + tabBuffer.String())
+	return nil
+}
+
+func Prepare(app *cli.Context) error {
+	var err error
+	if err = cmds.InitLogging(); err != nil {
+		return err
+	}
+	controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{
+		Stage: pointer.StringPtr(secretsencrypt.EncryptionPrepare),
+		Force: controlConfig.EncryptForce,
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("prepare completed successfully")
+	return nil
+}
+
+func Rotate(app *cli.Context) error {
+	if err := cmds.InitLogging(); err != nil {
+		return err
+	}
+	controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{
+		Stage: pointer.StringPtr(secretsencrypt.EncryptionRotate),
+		Force: controlConfig.EncryptForce,
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("rotate completed successfully")
+	return nil
+}
+
+func Reencrypt(app *cli.Context) error {
+	var err error
+	if err = cmds.InitLogging(); err != nil {
+		return err
+	}
+	controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{
+		Stage: pointer.StringPtr(secretsencrypt.EncryptionReencryptActive),
+		Force: controlConfig.EncryptForce,
+		Skip:  controlConfig.EncryptSkip,
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return err
+	}
+	fmt.Println("reencryption started")
+	return nil
+}

pkg/clientaccess/token.go

@@ -1,6 +1,7 @@
 package clientaccess
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
@@ -186,6 +187,16 @@ func (i *Info) Get(path string) ([]byte, error) {
 	return get(u.String(), GetHTTPClient(i.CACerts), i.Username, i.Password)
 }
 
+// Put makes a request to a subpath of info's BaseURL
+func (i *Info) Put(path string, body []byte) error {
+	u, err := url.Parse(i.BaseURL)
+	if err != nil {
+		return err
+	}
+	u.Path = path
+	return put(u.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password)
+}
+
 // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
 // and storing the CA bundle.
 func (i *Info) setServer(server string) error {
@@ -288,6 +299,32 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro
 	return ioutil.ReadAll(resp.Body)
 }
 
+// put makes a request to a url using a provided client, username, and password
+// only an error is returned
+func put(u string, body []byte, client *http.Client, username, password string) error {
+	req, err := http.NewRequest(http.MethodPut, u, bytes.NewBuffer(body))
+	if err != nil {
+		return err
+	}
+
+	if username != "" {
+		req.SetBasicAuth(username, password)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	respBody, _ := ioutil.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s: %s %s", u, resp.Status, string(respBody))
+	}
+
+	return nil
+}
+
 func FormatToken(token, certFile string) (string, error) {
 	if len(token) == 0 {
 		return token, nil

pkg/cluster/bootstrap.go

@@ -188,17 +188,18 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 		if err != nil {
 			return false, false, err
 		}
-
 		if isInitialized {
-			// If the database is initialized we skip bootstrapping; if the user wants to rejoin a
-			// cluster they need to delete the database.
-			logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName())
 			// This is a workaround for an issue that can be caused by terminating the cluster bootstrap before
 			// etcd is promoted from learner. Odds are we won't need this info, and we don't want to fail startup
 			// due to failure to retrieve it as this will break cold cluster restart, so we ignore any errors.
 			if c.config.JoinURL != "" && c.config.Token != "" {
 				c.clientAccessInfo, _ = clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
+				logrus.Infof("Joining %s cluster already initialized, forcing reconciliation", c.managedDB.EndpointName())
+				return true, true, nil
 			}
+			// If the database is initialized we skip bootstrapping; if the user wants to rejoin a
+			// cluster they need to delete the database.
+			logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName())
 			return false, true, nil
 		} else if c.config.JoinURL == "" {
 			// Not initialized, not joining - must be initializing (cluster-init)
@@ -353,7 +354,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 		if ec != nil {
 			etcdConfig = *ec
 		} else {
-			etcdConfig = c.etcdConfig
+			etcdConfig = c.EtcdConfig
 		}
 
 		storageClient, err := client.New(etcdConfig)
@@ -366,7 +367,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 
 	RETRY:
 		for {
-			value, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
+			value, c.saveBootstrap, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
 			if err != nil {
 				if strings.Contains(err.Error(), "not supported for learner") {
 					for range ticker.C {

pkg/cluster/bootstrap_test.go

@@ -130,7 +130,7 @@ func TestCluster_certDirsExist(t *testing.T) {
 				config:           tt.fields.config,
 				runtime:          tt.fields.runtime,
 				managedDB:        tt.fields.managedDB,
-				etcdConfig:       tt.fields.etcdConfig,
+				EtcdConfig:       tt.fields.etcdConfig,
 				storageStarted:   tt.fields.storageStarted,
 				saveBootstrap:    tt.fields.saveBootstrap,
 			}
@@ -240,7 +240,7 @@ func TestCluster_Snapshot(t *testing.T) {
 				config:           tt.fields.config,
 				runtime:          tt.fields.runtime,
 				managedDB:        tt.fields.managedDB,
-				etcdConfig:       tt.fields.etcdConfig,
+				EtcdConfig:       tt.fields.etcdConfig,
 				joining:          tt.fields.joining,
 				storageStarted:   tt.fields.storageStarted,
 				saveBootstrap:    tt.fields.saveBootstrap,

pkg/cluster/cluster.go

@@ -20,7 +20,7 @@ type Cluster struct {
 	config           *config.Control
 	runtime          *config.ControlRuntime
 	managedDB        managed.Driver
-	etcdConfig       endpoint.ETCDConfig
+	EtcdConfig       endpoint.ETCDConfig
 	joining          bool
 	storageStarted   bool
 	saveBootstrap    bool
@@ -85,7 +85,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 
 	// if necessary, store bootstrap data to datastore
 	if c.saveBootstrap {
-		if err := c.save(ctx, false); err != nil {
+		if err := Save(ctx, c.config, c.EtcdConfig, false); err != nil {
 			return nil, err
 		}
 	}
@@ -99,7 +99,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 			for {
 				select {
 				case <-ready:
-					if err := c.save(ctx, false); err != nil {
+					if err := Save(ctx, c.config, c.EtcdConfig, false); err != nil {
 						panic(err)
 					}
 
@@ -139,7 +139,7 @@ func (c *Cluster) startStorage(ctx context.Context) error {
 	// Persist the returned etcd configuration. We decide if we're doing leader election for embedded controllers
 	// based on what the kine wrapper tells us about the datastore. Single-node datastores like sqlite don't require
 	// leader election, while basically all others (etcd, external database, etc) do since they allow multiple servers.
-	c.etcdConfig = etcdConfig
+	c.EtcdConfig = etcdConfig
 	c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig
 	c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
 	c.config.NoLeaderElect = !etcdConfig.LeaderElect

pkg/cluster/storage.go

@@ -10,23 +10,25 @@ import (
 	"strings"
 
 	"github.com/k3s-io/kine/pkg/client"
+	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/rancher/k3s/pkg/bootstrap"
 	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 )
 
-// save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
+// Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
 // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token.
 // This is used when bootstrapping a cluster from a managed database or external etcd cluster.
 // This is NOT used with embedded etcd, which bootstraps over HTTP.
-func (c *Cluster) save(ctx context.Context, override bool) error {
+func Save(ctx context.Context, config *config.Control, etcdConfig endpoint.ETCDConfig, override bool) error {
 	buf := &bytes.Buffer{}
-	if err := bootstrap.ReadFromDisk(buf, &c.runtime.ControlRuntimeBootstrap); err != nil {
+	if err := bootstrap.ReadFromDisk(buf, &config.Runtime.ControlRuntimeBootstrap); err != nil {
 		return err
 	}
-	token := c.config.Token
+	token := config.Token
 	if token == "" {
-		tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir)
+		tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
 		if err != nil {
 			return err
 		}
@@ -42,12 +44,12 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
 		return err
 	}
 
-	storageClient, err := client.New(c.etcdConfig)
+	storageClient, err := client.New(etcdConfig)
 	if err != nil {
 		return err
 	}
 
-	if _, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil {
+	if _, _, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil {
 		return err
 	}
 
@@ -55,7 +57,7 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
 		if err.Error() == "key exists" {
 			logrus.Warn("bootstrap key already exists")
 			if override {
-				bsd, err := c.bootstrapKeyData(ctx, storageClient)
+				bsd, err := bootstrapKeyData(ctx, storageClient)
 				if err != nil {
 					return err
 				}
@@ -74,7 +76,7 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
 
 // bootstrapKeyData lists keys stored in the datastore with the prefix "/bootstrap", and
 // will return the first such key. It will return an error if not exactly one key is found.
-func (c *Cluster) bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) {
+func bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) {
 	bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
 	if err != nil {
 		return nil, err
@@ -96,7 +98,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 		return err
 	}
 
-	storageClient, err := client.New(c.etcdConfig)
+	storageClient, err := client.New(c.EtcdConfig)
 	if err != nil {
 		return err
 	}
@@ -119,7 +121,8 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 		return err
 	}
 
-	value, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
+	value, saveBootstrap, err := getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
+	c.saveBootstrap = saveBootstrap
 	if err != nil {
 		return err
 	}
@@ -139,38 +142,37 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 // hashed with empty string and will check for any key that is hashed by different token than the one
 // passed to it, it will return error if it finds a key that is hashed with different token and will return
 // value if it finds the key hashed by passed token or empty string
-func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, error) {
+func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, bool, error) {
 	emptyStringKey := storageKey("")
 	tokenKey := storageKey(normalizedToken)
 	bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	if len(bootstrapList) == 0 {
-		c.saveBootstrap = true
+		return nil, true, nil
-		return nil, nil
 	}
 	if len(bootstrapList) > 1 {
 		logrus.Warn("found multiple bootstrap keys in storage")
 	}
 	// check for empty string key and for old token format with k10 prefix
-	if err := c.migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
+	if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	// getting the list of bootstrap again after migrating the empty key
 	bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	for _, bootstrapKV := range bootstrapList {
 		// ensure bootstrap is stored in the current token's key
 		if string(bootstrapKV.Key) == tokenKey {
-			return &bootstrapKV, nil
+			return &bootstrapKV, false, nil
 		}
 	}
 
-	return nil, errors.New("bootstrap data already found and encrypted with different token")
+	return nil, false, errors.New("bootstrap data already found and encrypted with different token")
 }
 
 // readTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
@@ -209,7 +211,7 @@ func normalizeToken(token string) (string, error) {
 // migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is
 // hashed with empty string and keys that is hashed with old token format before normalizing
 // then migrate those and resave only with the normalized token
-func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
+func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
 	oldTokenKey := storageKey(oldToken)
 
 	for _, bootstrapKV := range bootstrapList {

pkg/daemons/config/types.go

@@ -155,6 +155,8 @@ type Control struct {
 	ClusterReset             bool
 	ClusterResetRestorePath  string
 	EncryptSecrets           bool
+	EncryptForce             bool
+	EncryptSkip              bool
 	TLSMinVersion            uint16
 	TLSCipherSuites          []uint16
 	EtcdSnapshotName         string
@@ -197,6 +199,7 @@ type ControlRuntimeBootstrap struct {
 	RequestHeaderCAKey string
 	IPSECKey           string
 	EncryptionConfig   string
+	EncryptionHash     string
 }
 
 type ControlRuntime struct {
@@ -254,6 +257,7 @@ type ControlRuntime struct {
 	ClientETCDKey            string
 
 	Core       *core.Factory
+	EtcdConfig endpoint.ETCDConfig
 }
 
 type ArgString []string

pkg/daemons/control/deps/deps.go

@@ -3,8 +3,10 @@ package deps
 import (
 	"crypto"
 	cryptorand "crypto/rand"
+	"crypto/sha256"
 	"crypto/x509"
 	b64 "encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -147,6 +149,7 @@ func CreateRuntimeCertFiles(config *config.Control, runtime *config.ControlRunti
 
 	if config.EncryptSecrets {
 		runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json")
+		runtime.EncryptionHash = filepath.Join(config.DataDir, "cred", "encryption-state.json")
 	}
 }
 
@@ -169,7 +172,7 @@ func GenServerDeps(config *config.Control, runtime *config.ControlRuntime) error
 		return err
 	}
 
-	if err := genEncryptionConfig(config, runtime); err != nil {
+	if err := genEncryptionConfigAndState(config, runtime); err != nil {
 		return err
 	}
 
@@ -649,7 +652,7 @@ func expired(certFile string, pool *x509.CertPool) bool {
 	return certutil.IsCertExpired(certificates[0], config.CertificateRenewDays)
 }
 
-func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlRuntime) error {
+func genEncryptionConfigAndState(controlConfig *config.Control, runtime *config.ControlRuntime) error {
 	if !controlConfig.EncryptSecrets {
 		return nil
 	}
@@ -690,9 +693,14 @@ func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlR
 			},
 		},
 	}
-	jsonfile, err := json.Marshal(encConfig)
+	b, err := json.Marshal(encConfig)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
+	if err := ioutil.WriteFile(runtime.EncryptionConfig, b, 0600); err != nil {
+		return err
+	}
+	encryptionConfigHash := sha256.Sum256(b)
+	ann := "start-" + hex.EncodeToString(encryptionConfigHash[:])
+	return ioutil.WriteFile(controlConfig.Runtime.EncryptionHash, []byte(ann), 0600)
 }

pkg/daemons/control/server.go

@@ -260,6 +260,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	}
 
 	runtime.ETCDReady = ready
+	runtime.EtcdConfig = cluster.EtcdConfig
 	return nil
 }
 

pkg/deploy/controller.go

@@ -18,10 +18,10 @@ import (
 	"github.com/rancher/k3s/pkg/agent/util"
 	apisv1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1"
 	controllersv1 "github.com/rancher/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
+	pkgutil "github.com/rancher/k3s/pkg/util"
 	"github.com/rancher/wrangler/pkg/apply"
 	"github.com/rancher/wrangler/pkg/merr"
 	"github.com/rancher/wrangler/pkg/objectset"
-	"github.com/rancher/wrangler/pkg/schemes"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -31,7 +31,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	yamlDecoder "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/kubernetes"
-	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/record"
 )
 
@@ -74,12 +73,7 @@ type watcher struct {
 
 // start calls listFiles at regular intervals to trigger application of manifests that have changed on disk.
 func (w *watcher) start(ctx context.Context, client kubernetes.Interface) {
-	nodeName := os.Getenv("NODE_NAME")
+	w.recorder = pkgutil.BuildControllerEventRecorder(client, ControllerName)
-	broadcaster := record.NewBroadcaster()
-	broadcaster.StartLogging(logrus.Infof)
-	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(metav1.NamespaceSystem)})
-	w.recorder = broadcaster.NewRecorder(schemes.All, corev1.EventSource{Component: ControllerName, Host: nodeName})
-
 	force := true
 	for {
 		if err := w.listFiles(force); err == nil {

pkg/secretsencrypt/config.go

@@ -0,0 +1,174 @@
+package secretsencrypt
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/version"
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+)
+
+const (
+	EncryptionStart             string = "start"
+	EncryptionPrepare           string = "prepare"
+	EncryptionRotate            string = "rotate"
+	EncryptionReencryptRequest  string = "reencrypt_request"
+	EncryptionReencryptActive   string = "reencrypt_active"
+	EncryptionReencryptFinished string = "reencrypt_finished"
+)
+
+var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash"
+
+func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1.ProviderConfiguration, error) {
+	curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	curEncryption := apiserverconfigv1.EncryptionConfiguration{}
+	if err = json.Unmarshal(curEncryptionByte, &curEncryption); err != nil {
+		return nil, err
+	}
+	return curEncryption.Resources[0].Providers, nil
+}
+
+func GetEncryptionKeys(runtime *config.ControlRuntime) ([]apiserverconfigv1.Key, error) {
+
+	providers, err := GetEncryptionProviders(runtime)
+	if err != nil {
+		return nil, err
+	}
+	if len(providers) > 2 {
+		return nil, fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
+	}
+
+	var curKeys []apiserverconfigv1.Key
+	for _, p := range providers {
+		if p.AESCBC != nil {
+			curKeys = append(curKeys, p.AESCBC.Keys...)
+		}
+		if p.AESGCM != nil || p.KMS != nil || p.Secretbox != nil {
+			return nil, fmt.Errorf("non-standard encryption keys found")
+		}
+	}
+	return curKeys, nil
+}
+
+func WriteEncryptionConfig(runtime *config.ControlRuntime, keys []apiserverconfigv1.Key, enable bool) error {
+
+	// Placing the identity provider first disables encryption
+	var providers []apiserverconfigv1.ProviderConfiguration
+	if enable {
+		providers = []apiserverconfigv1.ProviderConfiguration{
+			{
+				AESCBC: &apiserverconfigv1.AESConfiguration{
+					Keys: keys,
+				},
+			},
+			{
+				Identity: &apiserverconfigv1.IdentityConfiguration{},
+			},
+		}
+	} else {
+		providers = []apiserverconfigv1.ProviderConfiguration{
+			{
+				Identity: &apiserverconfigv1.IdentityConfiguration{},
+			},
+			{
+				AESCBC: &apiserverconfigv1.AESConfiguration{
+					Keys: keys,
+				},
+			},
+		}
+	}
+
+	encConfig := apiserverconfigv1.EncryptionConfiguration{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "EncryptionConfiguration",
+			APIVersion: "apiserver.config.k8s.io/v1",
+		},
+		Resources: []apiserverconfigv1.ResourceConfiguration{
+			{
+				Resources: []string{"secrets"},
+				Providers: providers,
+			},
+		},
+	}
+	jsonfile, err := json.Marshal(encConfig)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
+}
+
+func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) {
+	curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig)
+	if err != nil {
+		return "", err
+	}
+	encryptionConfigHash := sha256.Sum256(curEncryptionByte)
+	return hex.EncodeToString(encryptionConfigHash[:]), nil
+}
+
+// GenReencryptHash generates a sha256 hash fom the existing secrets keys and
+// a new key based on the input arguments.
+func GenReencryptHash(runtime *config.ControlRuntime, keyName string) (string, error) {
+
+	keys, err := GetEncryptionKeys(runtime)
+	if err != nil {
+		return "", err
+	}
+	newKey := apiserverconfigv1.Key{
+		Name:   keyName,
+		Secret: "12345",
+	}
+	keys = append(keys, newKey)
+	b, err := json.Marshal(keys)
+	if err != nil {
+		return "", err
+	}
+	hash := sha256.Sum256(b)
+	return hex.EncodeToString(hash[:]), nil
+}
+
+func getEncryptionHashFile(runtime *config.ControlRuntime) (string, error) {
+	curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionHash)
+	if err != nil {
+		return "", err
+	}
+	return string(curEncryptionByte), nil
+}
+
+func BootstrapEncryptionHashAnnotation(node *corev1.Node, runtime *config.ControlRuntime) error {
+	existingAnn, err := getEncryptionHashFile(runtime)
+	if err != nil {
+		return err
+	}
+	node.Annotations[EncryptionHashAnnotation] = existingAnn
+	return nil
+}
+
+func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, stage string) error {
+	encryptionConfigHash, err := GenEncryptionConfigHash(runtime)
+	if err != nil {
+		return err
+	}
+	if node.Annotations == nil {
+		return fmt.Errorf("node annotations do not exist for %s", node.ObjectMeta.Name)
+	}
+	ann := stage + "-" + encryptionConfigHash
+	node.Annotations[EncryptionHashAnnotation] = ann
+	if _, err = runtime.Core.Core().V1().Node().Update(node); err != nil {
+		return err
+	}
+	logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
+	return ioutil.WriteFile(runtime.EncryptionHash, []byte(ann), 0600)
+}

pkg/secretsencrypt/controller.go

@@ -0,0 +1,196 @@
+package secretsencrypt
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/rancher/k3s/pkg/cluster"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/util"
+	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/pager"
+	"k8s.io/client-go/tools/record"
+)
+
+const (
+	controllerAgentName        string = "reencrypt-controller"
+	secretsUpdateStartEvent    string = "SecretsUpdateStart"
+	secretsProgressEvent       string = "SecretsProgress"
+	secretsUpdateCompleteEvent string = "SecretsUpdateComplete"
+	secretsUpdateErrorEvent    string = "SecretsUpdateError"
+)
+
+type handler struct {
+	ctx           context.Context
+	controlConfig *config.Control
+	nodes         coreclient.NodeController
+	secrets       coreclient.SecretController
+	recorder      record.EventRecorder
+}
+
+func Register(
+	ctx context.Context,
+	k8s kubernetes.Interface,
+	controlConfig *config.Control,
+	nodes coreclient.NodeController,
+	secrets coreclient.SecretController,
+) error {
+	h := &handler{
+		ctx:           ctx,
+		controlConfig: controlConfig,
+		nodes:         nodes,
+		secrets:       secrets,
+		recorder:      util.BuildControllerEventRecorder(k8s, controllerAgentName),
+	}
+
+	nodes.OnChange(ctx, "reencrypt-controller", h.onChangeNode)
+	return nil
+}
+
+// onChangeNode handles changes to Nodes. We are looking for a specific annotation change
+func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, error) {
+	if node == nil {
+		return nil, nil
+	}
+
+	ann, ok := node.Annotations[EncryptionHashAnnotation]
+	if !ok {
+		return node, nil
+	}
+
+	if valid, err := h.validateReencryptStage(node, ann); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	} else if !valid {
+		return node, nil
+	}
+
+	reencryptHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive)
+	if err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	ann = EncryptionReencryptActive + "-" + reencryptHash
+	node.Annotations[EncryptionHashAnnotation] = ann
+	node, err = h.nodes.Update(node)
+	if err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+
+	if err := h.updateSecrets(node); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+
+	// If skipping, revert back to the previous stage
+	if h.controlConfig.EncryptSkip {
+		BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime)
+		if node, err := h.nodes.Update(node); err != nil {
+			return node, err
+		}
+		return node, nil
+	}
+
+	// Remove last key
+	curKeys, err := GetEncryptionKeys(h.controlConfig.Runtime)
+	if err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+
+	curKeys = curKeys[:len(curKeys)-1]
+	if err = WriteEncryptionConfig(h.controlConfig.Runtime, curKeys, true); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	logrus.Infoln("Removed key: ", curKeys[len(curKeys)-1])
+	if err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	if err := WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	if err := cluster.Save(h.ctx, h.controlConfig, h.controlConfig.Runtime.EtcdConfig, true); err != nil {
+		h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
+		return node, err
+	}
+	return node, nil
+}
+
+// validateReencryptStage ensures that the request for reencryption is valid and
+// that there is only one active reencryption at a time
+func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (bool, error) {
+
+	split := strings.Split(annotation, "-")
+	if len(split) != 2 {
+		err := fmt.Errorf("invalid annotation %s found on node %s", annotation, node.ObjectMeta.Name)
+		return false, err
+	}
+	stage := split[0]
+	hash := split[1]
+
+	// Validate the specific stage and the request via sha256 hash
+	if stage != EncryptionReencryptRequest {
+		return false, nil
+	}
+	if reencryptRequestHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptRequest); err != nil {
+		return false, err
+	} else if reencryptRequestHash != hash {
+		err = fmt.Errorf("invalid hash: %s found on node %s", hash, node.ObjectMeta.Name)
+		return false, err
+	}
+
+	nodes, err := h.nodes.List(metav1.ListOptions{})
+	if err != nil {
+		return false, err
+	}
+	reencryptActiveHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive)
+	if err != nil {
+		return false, err
+	}
+	for _, node := range nodes.Items {
+		if ann, ok := node.Annotations[EncryptionHashAnnotation]; ok {
+			split := strings.Split(ann, "-")
+			if len(split) != 2 {
+				return false, fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name)
+			}
+			stage := split[0]
+			hash := split[1]
+			if stage == EncryptionReencryptActive && hash == reencryptActiveHash {
+				return false, fmt.Errorf("another reencrypt is already active")
+			}
+		}
+	}
+	return true, nil
+}
+
+func (h *handler) updateSecrets(node *corev1.Node) error {
+	secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
+		return h.secrets.List("", opts)
+	}))
+	i := 0
+	secretPager.EachListItem(h.ctx, metav1.ListOptions{}, func(obj runtime.Object) error {
+		if secret, ok := obj.(*corev1.Secret); ok {
+			if _, err := h.secrets.Update(secret); err != nil {
+				return fmt.Errorf("failed to reencrypted secret: %v", err)
+			}
+			if i != 0 && i%10 == 0 {
+				h.recorder.Eventf(node, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i)
+			}
+			i++
+		}
+		return nil
+	})
+	h.recorder.Eventf(node, corev1.EventTypeNormal, secretsUpdateCompleteEvent, "completed reencrypt of %d secrets", i)
+	return nil
+}

pkg/server/router.go

@@ -48,6 +48,8 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 	authed.Path(prefix + "/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
 	authed.Path(prefix + "/config").Handler(configHandler(serverConfig, cfg))
 	authed.Path(prefix + "/readyz").Handler(readyzHandler(serverConfig))
+	authed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig))
+	authed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig))
 
 	nodeAuthed := mux.NewRouter()
 	nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes"))

pkg/server/secrets-encrypt.go

@@ -0,0 +1,379 @@
+package server
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/rancher/k3s/pkg/cluster"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/secretsencrypt"
+	"github.com/rancher/wrangler/pkg/generated/controllers/core"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+	"k8s.io/utils/pointer"
+)
+
+const aescbcKeySize = 32
+
+type EncryptionState struct {
+	Stage        string   `json:"stage"`
+	ActiveKey    string   `json:"activekey"`
+	Enable       *bool    `json:"enable,omitempty"`
+	HashMatch    bool     `json:"hashmatch,omitempty"`
+	HashError    string   `json:"hasherror,omitempty"`
+	InactiveKeys []string `json:"inactivekeys,omitempty"`
+}
+
+type EncryptionRequest struct {
+	Stage  *string `json:"stage,omitempty"`
+	Enable *bool   `json:"enable,omitempty"`
+	Force  bool    `json:"force"`
+	Skip   bool    `json:"skip"`
+}
+
+func getEncryptionRequest(req *http.Request) (EncryptionRequest, error) {
+	b, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		return EncryptionRequest{}, err
+	}
+	result := EncryptionRequest{}
+	err = json.Unmarshal(b, &result)
+	return result, err
+}
+
+func encryptionStatusHandler(server *config.Control) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil {
+			resp.WriteHeader(http.StatusNotFound)
+			return
+		}
+		status, err := encryptionStatus(server)
+		if err != nil {
+			genErrorMessage(resp, http.StatusInternalServerError, err)
+			return
+		}
+		b, err := json.Marshal(status)
+		if err != nil {
+			genErrorMessage(resp, http.StatusInternalServerError, err)
+			return
+		}
+		resp.Write(b)
+	})
+}
+
+func encryptionStatus(server *config.Control) (EncryptionState, error) {
+	state := EncryptionState{}
+	providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime)
+	if os.IsNotExist(err) {
+		return state, nil
+	} else if err != nil {
+		return state, err
+	}
+	if providers[1].Identity != nil && providers[0].AESCBC != nil {
+		state.Enable = pointer.Bool(true)
+	} else if providers[0].Identity != nil && providers[1].AESCBC != nil || !server.EncryptSecrets {
+		state.Enable = pointer.Bool(false)
+	}
+
+	if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), ""); err != nil {
+		state.HashMatch = false
+		state.HashError = err.Error()
+	} else {
+		state.HashMatch = true
+	}
+	stage, _, err := getEncryptionHashAnnotation(server.Runtime.Core.Core())
+	if err != nil {
+		return state, err
+	}
+	state.Stage = stage
+	active := true
+	for _, p := range providers {
+		if p.AESCBC != nil {
+			for _, aesKey := range p.AESCBC.Keys {
+				if active {
+					active = false
+					state.ActiveKey = aesKey.Name
+				} else {
+					state.InactiveKeys = append(state.InactiveKeys, aesKey.Name)
+				}
+			}
+		}
+		if p.Identity != nil {
+			active = false
+		}
+	}
+
+	return state, nil
+}
+
+func encryptionEnable(ctx context.Context, server *config.Control, enable bool) error {
+	providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime)
+	if err != nil {
+		return err
+	}
+	if len(providers) > 2 {
+		return fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
+	}
+	curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
+	if err != nil {
+		return err
+	}
+	if providers[1].Identity != nil && providers[0].AESCBC != nil && !enable {
+		logrus.Infoln("Disabling secrets encryption")
+		if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
+			return err
+		}
+	} else if !enable {
+		logrus.Infoln("Secrets encryption already disabled")
+		return nil
+	} else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable {
+		logrus.Infoln("Enabling secrets encryption")
+		if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
+			return err
+		}
+	} else if enable {
+		logrus.Infoln("Secrets encryption already enabled")
+		return nil
+	} else {
+		return fmt.Errorf("unable to enable/disable secrets encryption, unknown configuration")
+	}
+	return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
+}
+
+func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil {
+			resp.WriteHeader(http.StatusNotFound)
+			return
+		}
+		if req.Method != http.MethodPut {
+			resp.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		encryptReq, err := getEncryptionRequest(req)
+		if err != nil {
+			resp.WriteHeader(http.StatusBadRequest)
+			resp.Write([]byte(err.Error()))
+			return
+		}
+		if encryptReq.Stage != nil {
+			switch *encryptReq.Stage {
+			case secretsencrypt.EncryptionPrepare:
+				err = encryptionPrepare(ctx, server, encryptReq.Force)
+			case secretsencrypt.EncryptionRotate:
+				err = encryptionRotate(ctx, server, encryptReq.Force)
+			case secretsencrypt.EncryptionReencryptActive:
+				err = encryptionReencrypt(ctx, server, encryptReq.Force, encryptReq.Skip)
+			default:
+				err = fmt.Errorf("unknown stage %s requested", *encryptReq.Stage)
+			}
+		} else if encryptReq.Enable != nil {
+			err = encryptionEnable(ctx, server, *encryptReq.Enable)
+		}
+
+		if err != nil {
+			genErrorMessage(resp, http.StatusBadRequest, err)
+			return
+		}
+		resp.WriteHeader(http.StatusOK)
+	})
+}
+
+func encryptionPrepare(ctx context.Context, server *config.Control, force bool) error {
+	states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished
+	if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil && !force {
+		return err
+	}
+
+	curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
+	if err != nil {
+		return err
+	}
+
+	if err := AppendNewEncryptionKey(&curKeys); err != nil {
+		return err
+	}
+	logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1])
+
+	if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil {
+		return err
+	}
+	nodeName := os.Getenv("NODE_NAME")
+	node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if err = secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare); err != nil {
+		return err
+	}
+	return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
+}
+
+func encryptionRotate(ctx context.Context, server *config.Control, force bool) error {
+
+	if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionPrepare); err != nil && !force {
+		return err
+	}
+
+	curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
+	if err != nil {
+		return err
+	}
+
+	// Right rotate elements
+	rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...)
+
+	if err = secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true); err != nil {
+		return err
+	}
+	logrus.Infoln("Encryption keys right rotated")
+	nodeName := os.Getenv("NODE_NAME")
+	node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if err := secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate); err != nil {
+		return err
+	}
+	return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
+}
+
+func encryptionReencrypt(ctx context.Context, server *config.Control, force bool, skip bool) error {
+
+	if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionRotate); err != nil && !force {
+		return err
+	}
+	server.EncryptForce = force
+	server.EncryptSkip = skip
+	nodeName := os.Getenv("NODE_NAME")
+	node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest)
+	if err != nil {
+		return err
+	}
+	ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash
+	node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann
+	if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil {
+		return err
+	}
+	logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
+	return nil
+}
+
+func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error {
+
+	aescbcKey := make([]byte, aescbcKeySize)
+	_, err := rand.Read(aescbcKey)
+	if err != nil {
+		return err
+	}
+	encodedKey := base64.StdEncoding.EncodeToString(aescbcKey)
+
+	newKey := []apiserverconfigv1.Key{
+		{
+			Name:   "aescbckey-" + time.Now().Format(time.RFC3339),
+			Secret: encodedKey,
+		},
+	}
+	*keys = append(*keys, newKey...)
+	return nil
+}
+
+func getEncryptionHashAnnotation(core core.Interface) (string, string, error) {
+	nodeName := os.Getenv("NODE_NAME")
+	node, err := core.V1().Node().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		return "", "", err
+	}
+	ann := node.Annotations[secretsencrypt.EncryptionHashAnnotation]
+	split := strings.Split(ann, "-")
+	if len(split) != 2 {
+		return "", "", fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name)
+	}
+	return split[0], split[1], nil
+}
+
+func getServerNodes(core core.Interface) ([]corev1.Node, error) {
+	nodes, err := core.V1().Node().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	var serverNodes []corev1.Node
+	for _, node := range nodes.Items {
+		if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" {
+			serverNodes = append(serverNodes, node)
+		}
+	}
+	return serverNodes, nil
+}
+
+// verifyEncryptionHashAnnotation checks that all nodes are on the same stage,
+// and that a request for new stage is valid
+func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.Interface, prevStage string) error {
+
+	var firstHash string
+	var firstNodeName string
+	first := true
+	serverNodes, err := getServerNodes(core)
+	if err != nil {
+		return err
+	}
+	for _, node := range serverNodes {
+		hash, ok := node.Annotations[secretsencrypt.EncryptionHashAnnotation]
+		if ok && first {
+			firstHash = hash
+			first = false
+			firstNodeName = node.ObjectMeta.Name
+		} else if ok && hash != firstHash {
+			return fmt.Errorf("hash does not match between %s and %s", firstNodeName, node.ObjectMeta.Name)
+		}
+	}
+
+	if prevStage == "" {
+		return nil
+	}
+
+	oldStage, oldHash, err := getEncryptionHashAnnotation(core)
+	if err != nil {
+		return err
+	}
+
+	encryptionConfigHash, err := secretsencrypt.GenEncryptionConfigHash(runtime)
+	if err != nil {
+		return err
+	}
+	if !strings.Contains(prevStage, oldStage) {
+		return fmt.Errorf("incorrect stage: %s found on node %s", oldStage, serverNodes[0].ObjectMeta.Name)
+	} else if oldHash != encryptionConfigHash {
+		return fmt.Errorf("invalid hash: %s found on node %s", oldHash, serverNodes[0].ObjectMeta.Name)
+	}
+
+	return nil
+}
+
+func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error) {
+	errID, err := rand.Int(rand.Reader, big.NewInt(99999))
+	if err != nil {
+		resp.WriteHeader(http.StatusInternalServerError)
+		resp.Write([]byte(err.Error()))
+		return
+	}
+	logrus.Warnf("secrets-encrypt-%s: %s", errID.String(), passedErr.Error())
+	resp.WriteHeader(statusCode)
+	resp.Write([]byte(fmt.Sprintf("error secrets-encrypt-%s: see server logs for more info", errID.String())))
+}

pkg/server/server.go

@@ -25,6 +25,7 @@ import (
 	"github.com/rancher/k3s/pkg/node"
 	"github.com/rancher/k3s/pkg/nodepassword"
 	"github.com/rancher/k3s/pkg/rootlessports"
+	"github.com/rancher/k3s/pkg/secretsencrypt"
 	"github.com/rancher/k3s/pkg/servicelb"
 	"github.com/rancher/k3s/pkg/static"
 	"github.com/rancher/k3s/pkg/util"
@@ -169,7 +170,7 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err
 		}
 	}
 
-	go setControlPlaneRoleLabel(ctx, sc.Core.Core().V1().Node(), config)
+	go setNodeLabelsAndAnnotations(ctx, sc.Core.Core().V1().Node(), config)
 
 	go setClusterDNSConfig(ctx, config, sc.Core.Core().V1().ConfigMap())
 
@@ -232,6 +233,16 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
 		return err
 	}
 
+	if config.ControlConfig.EncryptSecrets {
+		if err := secretsencrypt.Register(ctx,
+			sc.K8s,
+			&config.ControlConfig,
+			sc.Core.Core().V1().Node(),
+			sc.Core.Core().V1().Secret()); err != nil {
+			return err
+		}
+	}
+
 	if config.Rootless {
 		return rootlessports.Register(ctx,
 			sc.Core.Core().V1().Service(),
@@ -490,7 +501,7 @@ func isSymlink(config string) bool {
 	return false
 }
 
-func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient, config *Config) error {
+func setNodeLabelsAndAnnotations(ctx context.Context, nodes v1.NodeClient, config *Config) error {
 	if config.DisableAgent || config.ControlConfig.DisableAPIServer {
 		return nil
 	}
@@ -515,18 +526,25 @@ func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient, config *
 				etcdRoleLabelExists = true
 			}
 		}
-		if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" && !etcdRoleLabelExists {
-			break
-		}
 		if node.Labels == nil {
 			node.Labels = make(map[string]string)
 		}
+		v, ok := node.Labels[ControlPlaneRoleLabelKey]
+		if !ok || v != "true" || etcdRoleLabelExists {
 			node.Labels[ControlPlaneRoleLabelKey] = "true"
 			node.Labels[MasterRoleLabelKey] = "true"
+		}
+
+		if config.ControlConfig.EncryptSecrets {
+			if err = secretsencrypt.BootstrapEncryptionHashAnnotation(node, config.ControlConfig.Runtime); err != nil {
+				logrus.Infof("Unable to set encryption hash annotation %s", err.Error())
+				break
+			}
+		}
 
 		_, err = nodes.Update(node)
 		if err == nil {
-			logrus.Infof("Control-plane role label has been set successfully on node: %s", nodeName)
+			logrus.Infof("Labels and annotations have been set successfully on node: %s", nodeName)
 			break
 		}
 		select {

pkg/util/api.go

@@ -5,15 +5,20 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/merr"
+	"github.com/rancher/wrangler/pkg/schemes"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	coregetter "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/record"
 )
 
 // This sets a default duration to wait for the apiserver to become ready. This is primarily used to
@@ -72,3 +77,12 @@ func WaitForAPIServerReady(ctx context.Context, client clientset.Interface, time
 
 	return nil
 }
+
+func BuildControllerEventRecorder(k8s clientset.Interface, controllerName string) record.EventRecorder {
+	logrus.Infof("Creating %s event broadcaster", controllerName)
+	eventBroadcaster := record.NewBroadcaster()
+	eventBroadcaster.StartLogging(logrus.Infof)
+	eventBroadcaster.StartRecordingToSink(&coregetter.EventSinkImpl{Interface: k8s.CoreV1().Events(metav1.NamespaceSystem)})
+	nodeName := os.Getenv("NODE_NAME")
+	return eventBroadcaster.NewRecorder(schemes.All, v1.EventSource{Component: controllerName, Host: nodeName})
+}

scripts/build

@@ -78,6 +78,7 @@ rm -f \
     bin/containerd-shim-runc-v2 \
     bin/k3s-server \
     bin/k3s-etcd-snapshot \
+    bin/k3s-secrets-encrypt \
     bin/k3s-certificate \
     bin/kubectl \
     bin/crictl \
@@ -108,6 +109,7 @@ CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STAT
 ln -s containerd ./bin/k3s-agent
 ln -s containerd ./bin/k3s-server
 ln -s containerd ./bin/k3s-etcd-snapshot
+ln -s containerd ./bin/k3s-secrets-encrypt
 ln -s containerd ./bin/k3s-certificate
 ln -s containerd ./bin/kubectl
 ln -s containerd ./bin/crictl

scripts/package-cli

@@ -7,7 +7,7 @@ cd $(dirname $0)/..
 
 GO=${GO-go}
 
-for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-certificate k3s ; do
+for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s ; do
     rm -f bin/$i
     ln -s containerd bin/$i
 done

tests/integration/localstorage/localstorage_int_test.go

@@ -15,7 +15,6 @@ import (
 
 var localStorageServer *testutil.K3sServer
 var localStorageServerArgs = []string{"--cluster-init"}
-var testDataDir = "../../testdata/"
 var _ = BeforeSuite(func() {
 	if !testutil.IsExistingServer() {
 		var err error
@@ -37,12 +36,12 @@ var _ = Describe("local storage", func() {
 			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 		})
 		It("creates a new pvc", func() {
-			result, err := testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pvc.yaml")
+			result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pvc.yaml")
 			Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created"))
 			Expect(err).NotTo(HaveOccurred())
 		})
 		It("creates a new pod", func() {
-			Expect(testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pod.yaml")).
+			Expect(testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pod.yaml")).
 				To(ContainSubstring("pod/volume-test created"))
 		})
 		It("shows storage up in kubectl", func() {
@@ -51,7 +50,7 @@ var _ = Describe("local storage", func() {
 			}, "45s", "1s").Should(MatchRegexp(`local-path-pvc.+Bound`))
 			Eventually(func() (string, error) {
 				return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv")
-			}, "10s", "1s").Should(MatchRegexp(`pvc.+2Gi.+Bound`))
+			}, "10s", "1s").Should(MatchRegexp(`pvc.+1Gi.+Bound`))
 			Eventually(func() (string, error) {
 				return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pod")
 			}, "10s", "1s").Should(MatchRegexp(`volume-test.+Running`))

tests/integration/localstorage/testdata/localstorage_pvc.yaml

@@ -9,4 +9,4 @@ spec:
   storageClassName: local-path
   resources:
     requests:
-      storage: 2Gi
+      storage: 1Gi

tests/integration/secretsencryption/secretsencryption_int_test.go

@@ -0,0 +1,155 @@
+package integration
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/reporters"
+	. "github.com/onsi/gomega"
+	testutil "github.com/rancher/k3s/tests/util"
+)
+
+var secretsEncryptionServer *testutil.K3sServer
+var secretsEncryptionDataDir = "/tmp/k3sse"
+
+var secretsEncryptionServerArgs = []string{"--secrets-encryption", "-d", secretsEncryptionDataDir}
+var _ = BeforeSuite(func() {
+	if !testutil.IsExistingServer() {
+		var err error
+		Expect(os.MkdirAll(secretsEncryptionDataDir, 0777)).To(Succeed())
+		secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+		Expect(err).ToNot(HaveOccurred())
+	}
+})
+
+var _ = Describe("secrets encryption rotation", func() {
+	BeforeEach(func() {
+		if testutil.IsExistingServer() {
+			Skip("Test does not support running on existing k3s servers")
+		}
+	})
+	When("A server starts with secrets encryption", func() {
+		It("starts up with no problems", func() {
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+		})
+		It("it creates a encryption key", func() {
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Encryption Status: Enabled"))
+			Expect(result).To(ContainSubstring("Current Rotation Stage: start"))
+		})
+	})
+	When("A server rotates encryption keys", func() {
+		It("it prepares to rotate", func() {
+			Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)).
+				To(ContainSubstring("prepare completed successfully"))
+
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Current Rotation Stage: prepare"))
+			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
+			Expect(err).ToNot(HaveOccurred())
+			keys := reg.FindAllString(result, -1)
+			Expect(keys).To(HaveLen(2))
+			Expect(keys[0]).To(ContainSubstring("aescbckey"))
+			Expect(keys[1]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
+		})
+		It("restarts the server", func() {
+			var err error
+			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+		})
+		It("rotates the keys", func() {
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir)
+			}, "10s", "2s").Should(ContainSubstring("rotate completed successfully"))
+
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Current Rotation Stage: rotate"))
+			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
+			Expect(err).ToNot(HaveOccurred())
+			keys := reg.FindAllString(result, -1)
+			Expect(keys).To(HaveLen(2))
+			Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
+			Expect(keys[1]).To(ContainSubstring("aescbckey"))
+		})
+		It("restarts the server", func() {
+			var err error
+			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			time.Sleep(10 * time.Second)
+		})
+		It("reencrypts the keys", func() {
+			Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)).
+				To(ContainSubstring("reencryption started"))
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
+			Expect(err).ToNot(HaveOccurred())
+			keys := reg.FindAllString(result, -1)
+			Expect(keys).To(HaveLen(1))
+			Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
+		})
+	})
+	When("A server disables encryption", func() {
+		It("it triggers the disable", func() {
+			Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)).
+				To(ContainSubstring("secrets-encryption disabled"))
+
+			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
+		})
+		It("restarts the server", func() {
+			var err error
+			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
+			time.Sleep(10 * time.Second)
+		})
+		It("reencrypts the keys", func() {
+			result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("reencryption started"))
+
+			result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	if !testutil.IsExistingServer() {
+		Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+		Expect(testutil.K3sRemoveDataDir(secretsEncryptionDataDir)).To(Succeed())
+	}
+})
+
+func Test_IntegrationSecretsEncryption(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{
+		reporters.NewJUnitReporter("/tmp/results/junit-se.xml"),
+	})
+}

tests/util/cmd.go

@@ -80,6 +80,18 @@ func K3sCmd(cmdName string, cmdArgs ...string) (string, error) {
 	return string(byteOut), err
 }
 
+// K3sRemoveDataDir removes the provided directory as root
+func K3sRemoveDataDir(dataDir string) error {
+	var cmd *exec.Cmd
+	if IsRoot() {
+		cmd = exec.Command("rm", "-rf", dataDir)
+	} else {
+		cmd = exec.Command("sudo", "rm", "-rf", dataDir)
+	}
+	_, err := cmd.CombinedOutput()
+	return err
+}
+
 func contains(source []string, target string) bool {
 	for _, s := range source {
 		if s == target {
@@ -168,11 +180,5 @@ func K3sKillServer(server *K3sServer) error {
 			return err
 		}
 	}
-	if err := flock.Release(server.lock); err != nil {
+	return flock.Release(server.lock)
-		return err
-	}
-	if !flock.CheckLock(lockFile) {
-		return os.RemoveAll(lockFile)
-	}
-	return nil
 }