From bcb662926d005dd65275bc462d4664ff5994ce07 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 7 Dec 2021 14:31:32 -0800 Subject: [PATCH] Secrets-encryption rotation (#4372) * Regular CLI framework for encrypt commands * New secrets-encryption feature * New integration test * fixes for flaky integration test CI * Fix to bootstrap on restart of existing nodes * Consolidate event recorder Signed-off-by: Derek Nola --- .github/workflows/integration.yaml | 13 +- cmd/encrypt/main.go | 32 ++ cmd/k3s/main.go | 10 + cmd/server/main.go | 10 + main.go | 10 + pkg/cli/cmds/agent.go | 19 +- pkg/cli/cmds/etcd_snapshot.go | 6 +- pkg/cli/cmds/secrets_encrypt.go | 94 +++++ pkg/cli/cmds/server.go | 28 +- pkg/cli/secretsencrypt/secrets_encrypt.go | 216 ++++++++++ pkg/clientaccess/token.go | 37 ++ pkg/cluster/bootstrap.go | 13 +- pkg/cluster/bootstrap_test.go | 4 +- pkg/cluster/cluster.go | 8 +- pkg/cluster/storage.go | 44 +- pkg/daemons/config/types.go | 6 +- pkg/daemons/control/deps/deps.go | 16 +- pkg/daemons/control/server.go | 1 + pkg/deploy/controller.go | 10 +- pkg/secretsencrypt/config.go | 174 ++++++++ pkg/secretsencrypt/controller.go | 196 +++++++++ pkg/server/router.go | 2 + pkg/server/secrets-encrypt.go | 379 ++++++++++++++++++ pkg/server/server.go | 34 +- pkg/util/api.go | 14 + scripts/build | 2 + scripts/package-cli | 2 +- .../localstorage/localstorage_int_test.go | 7 +- .../testdata/localstorage_pod.yaml | 0 .../testdata/localstorage_pvc.yaml | 2 +- .../secretsencryption_int_test.go | 155 +++++++ tests/util/cmd.go | 20 +- 32 files changed, 1460 insertions(+), 104 deletions(-) create mode 100644 cmd/encrypt/main.go create mode 100644 pkg/cli/cmds/secrets_encrypt.go create mode 100644 pkg/cli/secretsencrypt/secrets_encrypt.go create mode 100644 pkg/secretsencrypt/config.go create mode 100644 pkg/secretsencrypt/controller.go create mode 100644 pkg/server/secrets-encrypt.go rename tests/{ => integration/localstorage}/testdata/localstorage_pod.yaml (100%) rename tests/{ => integration/localstorage}/testdata/localstorage_pvc.yaml (91%) create mode 100644 tests/integration/secretsencryption/secretsencryption_int_test.go diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 6ea8098708..b70ac9f1cf 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -56,17 +56,8 @@ jobs: - name: Run Integration Tests run: | chmod +x ./dist/artifacts/k3s - go test -coverpkg=./... -coverprofile=coverage.out ./pkg/... -run Integration - go tool cover -func coverage.out - # these tests do not relate to coverage and must be run separately - go test ./tests/integration/... -run Integration + go test ./pkg/... ./tests/integration/... -run Integration - name: On Failure, Launch Debug Session if: ${{ failure() }} uses: mxschmitt/action-tmate@v3 - timeout-minutes: 5 - - name: Upload Results To Codecov - uses: codecov/codecov-action@v1 - with: - files: ./coverage.out - flags: inttests # optional - verbose: true # optional (default = false) + timeout-minutes: 5 \ No newline at end of file diff --git a/cmd/encrypt/main.go b/cmd/encrypt/main.go new file mode 100644 index 0000000000..e1b43b83b2 --- /dev/null +++ b/cmd/encrypt/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "errors" + "os" + + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/cli/secretsencrypt" + "github.com/rancher/k3s/pkg/configfilearg" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +func main() { + app := cmds.NewApp() + app.Commands = []cli.Command{ + cmds.NewSecretsEncryptCommand(cli.ShowAppHelp, + cmds.NewSecretsEncryptSubcommands( + secretsencrypt.Status, + secretsencrypt.Enable, + secretsencrypt.Disable, + secretsencrypt.Prepare, + secretsencrypt.Rotate, + secretsencrypt.Reencrypt), + ), + } + + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { + logrus.Fatal(err) + } +} diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index bf764aff39..de9760d682 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -35,6 +35,7 @@ func main() { } etcdsnapshotCommand := internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args) + secretsencryptCommand := internalCLIAction(version.Program+"-"+cmds.SecretsEncryptCommand, dataDir, os.Args) certCommand := internalCLIAction(version.Program+"-"+cmds.CertCommand, dataDir, os.Args) // Handle subcommand invocation (k3s server, k3s crictl, etc) @@ -53,6 +54,15 @@ func main() { etcdsnapshotCommand, etcdsnapshotCommand), ), + cmds.NewSecretsEncryptCommand(secretsencryptCommand, + cmds.NewSecretsEncryptSubcommands( + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand, + secretsencryptCommand), + ), cmds.NewCertCommand( cmds.NewCertSubcommands( certCommand), diff --git a/cmd/server/main.go b/cmd/server/main.go index f6493880f3..0e3d8cee87 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -15,6 +15,7 @@ import ( "github.com/rancher/k3s/pkg/cli/ctr" "github.com/rancher/k3s/pkg/cli/etcdsnapshot" "github.com/rancher/k3s/pkg/cli/kubectl" + "github.com/rancher/k3s/pkg/cli/secretsencrypt" "github.com/rancher/k3s/pkg/cli/server" "github.com/rancher/k3s/pkg/configfilearg" "github.com/rancher/k3s/pkg/containerd" @@ -53,6 +54,15 @@ func main() { etcdsnapshot.Prune, etcdsnapshot.Run), ), + cmds.NewSecretsEncryptCommand(cli.ShowAppHelp, + cmds.NewSecretsEncryptSubcommands( + secretsencrypt.Status, + secretsencrypt.Enable, + secretsencrypt.Disable, + secretsencrypt.Prepare, + secretsencrypt.Rotate, + secretsencrypt.Reencrypt), + ), cmds.NewCertCommand( cmds.NewCertSubcommands( cert.Run), diff --git a/main.go b/main.go index 08241f93e7..1eb41188c4 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "github.com/rancher/k3s/pkg/cli/crictl" "github.com/rancher/k3s/pkg/cli/etcdsnapshot" "github.com/rancher/k3s/pkg/cli/kubectl" + "github.com/rancher/k3s/pkg/cli/secretsencrypt" "github.com/rancher/k3s/pkg/cli/server" "github.com/rancher/k3s/pkg/configfilearg" "github.com/sirupsen/logrus" @@ -37,6 +38,15 @@ func main() { etcdsnapshot.Prune, etcdsnapshot.Run), ), + cmds.NewSecretsEncryptCommand(cli.ShowAppHelp, + cmds.NewSecretsEncryptSubcommands( + secretsencrypt.Status, + secretsencrypt.Enable, + secretsencrypt.Disable, + secretsencrypt.Prepare, + secretsencrypt.Rotate, + secretsencrypt.Reencrypt), + ), cmds.NewCertCommand( cmds.NewCertSubcommands( cert.Run), diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 0bdf24c034..31f40bfffa 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -56,9 +56,15 @@ type AgentShared struct { } var ( - appName = filepath.Base(os.Args[0]) - AgentConfig Agent - NodeIPFlag = cli.StringSliceFlag{ + appName = filepath.Base(os.Args[0]) + AgentConfig Agent + AgentTokenFlag = cli.StringFlag{ + Name: "token,t", + Usage: "(cluster) Token to use for authentication", + EnvVar: version.ProgramUpper + "_TOKEN", + Destination: &AgentConfig.Token, + } + NodeIPFlag = cli.StringSliceFlag{ Name: "node-ip,i", Usage: "(agent/networking) IPv4/IPv6 addresses to advertise for node", Value: &AgentConfig.NodeIP, @@ -217,12 +223,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { VModule, LogFile, AlsoLogToStderr, - cli.StringFlag{ - Name: "token,t", - Usage: "(cluster) Token to use for authentication", - EnvVar: version.ProgramUpper + "_TOKEN", - Destination: &AgentConfig.Token, - }, + AgentTokenFlag, cli.StringFlag{ Name: "token-file", Usage: "(cluster) Token file to use for authentication", diff --git a/pkg/cli/cmds/etcd_snapshot.go b/pkg/cli/cmds/etcd_snapshot.go index 86dcbc0bd9..e1d5c7208a 100644 --- a/pkg/cli/cmds/etcd_snapshot.go +++ b/pkg/cli/cmds/etcd_snapshot.go @@ -20,11 +20,7 @@ var EtcdSnapshotFlags = []cli.Flag{ EnvVar: version.ProgramUpper + "_NODE_NAME", Destination: &AgentConfig.NodeName, }, - cli.StringFlag{ - Name: "data-dir,d", - Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root", - Destination: &ServerConfig.DataDir, - }, + DataDirFlag, &cli.StringFlag{ Name: "dir,etcd-snapshot-dir", Usage: "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)", diff --git a/pkg/cli/cmds/secrets_encrypt.go b/pkg/cli/cmds/secrets_encrypt.go new file mode 100644 index 0000000000..e507e95306 --- /dev/null +++ b/pkg/cli/cmds/secrets_encrypt.go @@ -0,0 +1,94 @@ +package cmds + +import ( + "github.com/urfave/cli" +) + +const SecretsEncryptCommand = "secrets-encrypt" + +var EncryptFlags = []cli.Flag{ + DataDirFlag, + ServerToken, +} + +func NewSecretsEncryptCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command { + return cli.Command{ + Name: SecretsEncryptCommand, + Usage: "Control secrets encryption and keys rotation", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: action, + Subcommands: subcommands, + } +} + +func NewSecretsEncryptSubcommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) []cli.Command { + return []cli.Command{ + { + Name: "status", + Usage: "Print current status of secrets encryption", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: status, + Flags: EncryptFlags, + }, + { + Name: "enable", + Usage: "Enable secrets encryption", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: enable, + Flags: EncryptFlags, + }, + { + Name: "disable", + Usage: "Disable secrets encryption", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: disable, + Flags: EncryptFlags, + }, + { + Name: "prepare", + Usage: "Prepare for encryption keys rotation", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: prepare, + Flags: append(EncryptFlags, &cli.BoolFlag{ + Name: "f,force", + Usage: "Force preparation.", + Destination: &ServerConfig.EncryptForce, + }), + }, + { + Name: "rotate", + Usage: "Rotate secrets encryption keys", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: rotate, + Flags: append(EncryptFlags, &cli.BoolFlag{ + Name: "f,force", + Usage: "Force key rotation.", + Destination: &ServerConfig.EncryptForce, + }), + }, + { + Name: "reencrypt", + Usage: "Reencrypt all data with new encryption key", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: reencrypt, + Flags: append(EncryptFlags, + &cli.BoolFlag{ + Name: "f,force", + Usage: "Force secrets reencryption.", + Destination: &ServerConfig.EncryptForce, + }, + &cli.BoolFlag{ + Name: "skip", + Usage: "Skip removing old key", + Destination: &ServerConfig.EncryptSkip, + }), + }, + } +} diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 5ee53b2d5e..237cc2a3b3 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -74,6 +74,8 @@ type Server struct { ClusterReset bool ClusterResetRestorePath string EncryptSecrets bool + EncryptForce bool + EncryptSkip bool SystemDefaultRegistry string StartupHooks []StartupHook EtcdSnapshotName string @@ -97,7 +99,18 @@ type Server struct { var ( ServerConfig Server - ClusterCIDR = cli.StringSliceFlag{ + DataDirFlag = cli.StringFlag{ + Name: "data-dir,d", + Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root", + Destination: &ServerConfig.DataDir, + } + ServerToken = cli.StringFlag{ + Name: "token,t", + Usage: "(cluster) Shared secret used to join a server or agent to a cluster", + Destination: &ServerConfig.Token, + EnvVar: version.ProgramUpper + "_TOKEN", + } + ClusterCIDR = cli.StringSliceFlag{ Name: "cluster-cidr", Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)", Value: &ServerConfig.ClusterCIDR, @@ -179,11 +192,7 @@ var ServerFlags = []cli.Flag{ Usage: "(listener) Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert", Value: &ServerConfig.TLSSan, }, - cli.StringFlag{ - Name: "data-dir,d", - Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root", - Destination: &ServerConfig.DataDir, - }, + DataDirFlag, ClusterCIDR, ServiceCIDR, ServiceNodePortRange, @@ -195,12 +204,7 @@ var ServerFlags = []cli.Flag{ Destination: &ServerConfig.FlannelBackend, Value: "vxlan", }, - cli.StringFlag{ - Name: "token,t", - Usage: "(cluster) Shared secret used to join a server or agent to a cluster", - Destination: &ServerConfig.Token, - EnvVar: version.ProgramUpper + "_TOKEN", - }, + ServerToken, cli.StringFlag{ Name: "token-file", Usage: "(cluster) File containing the cluster-secret/token", diff --git a/pkg/cli/secretsencrypt/secrets_encrypt.go b/pkg/cli/secretsencrypt/secrets_encrypt.go new file mode 100644 index 0000000000..197a70470c --- /dev/null +++ b/pkg/cli/secretsencrypt/secrets_encrypt.go @@ -0,0 +1,216 @@ +package secretsencrypt + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "text/tabwriter" + + "github.com/erikdubbelboer/gspt" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/clientaccess" + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/secretsencrypt" + "github.com/rancher/k3s/pkg/server" + "github.com/rancher/k3s/pkg/version" + "github.com/urfave/cli" + "k8s.io/utils/pointer" +) + +func commandPrep(app *cli.Context, cfg *cmds.Server) (config.Control, *clientaccess.Info, error) { + var controlConfig config.Control + var err error + // hide process arguments from ps output, since they may contain + // database credentials or other secrets. + gspt.SetProcTitle(os.Args[0] + " encrypt") + + controlConfig.DataDir, err = server.ResolveDataDir(cfg.DataDir) + if err != nil { + return controlConfig, nil, err + } + if cfg.ServerURL == "" { + cfg.ServerURL = "https://127.0.0.1:6443" + } + + if cfg.Token == "" { + fp := filepath.Join(controlConfig.DataDir, "token") + tokenByte, err := ioutil.ReadFile(fp) + if err != nil { + return controlConfig, nil, err + } + controlConfig.Token = string(bytes.TrimRight(tokenByte, "\n")) + } else { + controlConfig.Token = cfg.Token + } + controlConfig.EncryptForce = cfg.EncryptForce + controlConfig.EncryptSkip = cfg.EncryptSkip + info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, controlConfig.Token, "node") + if err != nil { + return controlConfig, nil, err + } + return controlConfig, info, nil +} + +func Enable(app *cli.Context) error { + var err error + if err = cmds.InitLogging(); err != nil { + return err + } + _, info, err := commandPrep(app, &cmds.ServerConfig) + if err != nil { + return err + } + b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(true)}) + if err != nil { + return err + } + if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { + return err + } + fmt.Println("secrets-encryption enabled") + return nil +} + +func Disable(app *cli.Context) error { + + if err := cmds.InitLogging(); err != nil { + return err + } + _, info, err := commandPrep(app, &cmds.ServerConfig) + if err != nil { + return err + } + b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(false)}) + if err != nil { + return err + } + if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { + return err + } + fmt.Println("secrets-encryption disabled") + return nil +} + +func Status(app *cli.Context) error { + if err := cmds.InitLogging(); err != nil { + return err + } + _, info, err := commandPrep(app, &cmds.ServerConfig) + if err != nil { + return err + } + data, err := info.Get("/v1-" + version.Program + "/encrypt/status") + if err != nil { + return err + } + status := server.EncryptionState{} + if err := json.Unmarshal(data, &status); err != nil { + return err + } + + if status.Enable == nil { + fmt.Println("Encryption Status: Disabled, no configuration file found") + return nil + } + + var statusOutput string + if *status.Enable { + statusOutput += "Encryption Status: Enabled\n" + } else { + statusOutput += "Encryption Status: Disabled\n" + } + statusOutput += fmt.Sprintln("Current Rotation Stage:", status.Stage) + + if status.HashMatch { + statusOutput += fmt.Sprintln("Server Encryption Hashes: All hashes match") + } else { + statusOutput += fmt.Sprintf("Server Encryption Hashes: %s\n", status.HashError) + } + + var tabBuffer bytes.Buffer + w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0) + fmt.Fprintf(w, "\n") + fmt.Fprintf(w, "Active\tKey Type\tName\n") + fmt.Fprintf(w, "------\t--------\t----\n") + if status.ActiveKey != "" { + fmt.Fprintf(w, " *\t%s\t%s\n", "AES-CBC", status.ActiveKey) + } + for _, k := range status.InactiveKeys { + fmt.Fprintf(w, "\t%s\t%s\n", "AES-CBC", k) + } + w.Flush() + fmt.Println(statusOutput + tabBuffer.String()) + return nil +} + +func Prepare(app *cli.Context) error { + var err error + if err = cmds.InitLogging(); err != nil { + return err + } + controlConfig, info, err := commandPrep(app, &cmds.ServerConfig) + if err != nil { + return err + } + b, err := json.Marshal(server.EncryptionRequest{ + Stage: pointer.StringPtr(secretsencrypt.EncryptionPrepare), + Force: controlConfig.EncryptForce, + }) + if err != nil { + return err + } + if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { + return err + } + fmt.Println("prepare completed successfully") + return nil +} + +func Rotate(app *cli.Context) error { + if err := cmds.InitLogging(); err != nil { + return err + } + controlConfig, info, err := commandPrep(app, &cmds.ServerConfig) + if err != nil { + return err + } + b, err := json.Marshal(server.EncryptionRequest{ + Stage: pointer.StringPtr(secretsencrypt.EncryptionRotate), + Force: controlConfig.EncryptForce, + }) + if err != nil { + return err + } + if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { + return err + } + fmt.Println("rotate completed successfully") + return nil +} + +func Reencrypt(app *cli.Context) error { + var err error + if err = cmds.InitLogging(); err != nil { + return err + } + controlConfig, info, err := commandPrep(app, &cmds.ServerConfig) + if err != nil { + return err + } + b, err := json.Marshal(server.EncryptionRequest{ + Stage: pointer.StringPtr(secretsencrypt.EncryptionReencryptActive), + Force: controlConfig.EncryptForce, + Skip: controlConfig.EncryptSkip, + }) + if err != nil { + return err + } + if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil { + return err + } + fmt.Println("reencryption started") + return nil +} diff --git a/pkg/clientaccess/token.go b/pkg/clientaccess/token.go index 5476f71874..57192fc56d 100644 --- a/pkg/clientaccess/token.go +++ b/pkg/clientaccess/token.go @@ -1,6 +1,7 @@ package clientaccess import ( + "bytes" "crypto/sha256" "crypto/tls" "crypto/x509" @@ -186,6 +187,16 @@ func (i *Info) Get(path string) ([]byte, error) { return get(u.String(), GetHTTPClient(i.CACerts), i.Username, i.Password) } +// Put makes a request to a subpath of info's BaseURL +func (i *Info) Put(path string, body []byte) error { + u, err := url.Parse(i.BaseURL) + if err != nil { + return err + } + u.Path = path + return put(u.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password) +} + // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server // and storing the CA bundle. func (i *Info) setServer(server string) error { @@ -288,6 +299,32 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro return ioutil.ReadAll(resp.Body) } +// put makes a request to a url using a provided client, username, and password +// only an error is returned +func put(u string, body []byte, client *http.Client, username, password string) error { + req, err := http.NewRequest(http.MethodPut, u, bytes.NewBuffer(body)) + if err != nil { + return err + } + + if username != "" { + req.SetBasicAuth(username, password) + } + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respBody, _ := ioutil.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s: %s %s", u, resp.Status, string(respBody)) + } + + return nil +} + func FormatToken(token, certFile string) (string, error) { if len(token) == 0 { return token, nil diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 53fa068ec6..7a82cedcb2 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -188,17 +188,18 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) { if err != nil { return false, false, err } - if isInitialized { - // If the database is initialized we skip bootstrapping; if the user wants to rejoin a - // cluster they need to delete the database. - logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName()) // This is a workaround for an issue that can be caused by terminating the cluster bootstrap before // etcd is promoted from learner. Odds are we won't need this info, and we don't want to fail startup // due to failure to retrieve it as this will break cold cluster restart, so we ignore any errors. if c.config.JoinURL != "" && c.config.Token != "" { c.clientAccessInfo, _ = clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server") + logrus.Infof("Joining %s cluster already initialized, forcing reconciliation", c.managedDB.EndpointName()) + return true, true, nil } + // If the database is initialized we skip bootstrapping; if the user wants to rejoin a + // cluster they need to delete the database. + logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName()) return false, true, nil } else if c.config.JoinURL == "" { // Not initialized, not joining - must be initializing (cluster-init) @@ -353,7 +354,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, if ec != nil { etcdConfig = *ec } else { - etcdConfig = c.etcdConfig + etcdConfig = c.EtcdConfig } storageClient, err := client.New(etcdConfig) @@ -366,7 +367,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, RETRY: for { - value, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) + value, c.saveBootstrap, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) if err != nil { if strings.Contains(err.Error(), "not supported for learner") { for range ticker.C { diff --git a/pkg/cluster/bootstrap_test.go b/pkg/cluster/bootstrap_test.go index 4250c8c8ae..c9d623ed18 100644 --- a/pkg/cluster/bootstrap_test.go +++ b/pkg/cluster/bootstrap_test.go @@ -130,7 +130,7 @@ func TestCluster_certDirsExist(t *testing.T) { config: tt.fields.config, runtime: tt.fields.runtime, managedDB: tt.fields.managedDB, - etcdConfig: tt.fields.etcdConfig, + EtcdConfig: tt.fields.etcdConfig, storageStarted: tt.fields.storageStarted, saveBootstrap: tt.fields.saveBootstrap, } @@ -240,7 +240,7 @@ func TestCluster_Snapshot(t *testing.T) { config: tt.fields.config, runtime: tt.fields.runtime, managedDB: tt.fields.managedDB, - etcdConfig: tt.fields.etcdConfig, + EtcdConfig: tt.fields.etcdConfig, joining: tt.fields.joining, storageStarted: tt.fields.storageStarted, saveBootstrap: tt.fields.saveBootstrap, diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 3fd9250e97..eb2a5a4c71 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -20,7 +20,7 @@ type Cluster struct { config *config.Control runtime *config.ControlRuntime managedDB managed.Driver - etcdConfig endpoint.ETCDConfig + EtcdConfig endpoint.ETCDConfig joining bool storageStarted bool saveBootstrap bool @@ -85,7 +85,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { // if necessary, store bootstrap data to datastore if c.saveBootstrap { - if err := c.save(ctx, false); err != nil { + if err := Save(ctx, c.config, c.EtcdConfig, false); err != nil { return nil, err } } @@ -99,7 +99,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { for { select { case <-ready: - if err := c.save(ctx, false); err != nil { + if err := Save(ctx, c.config, c.EtcdConfig, false); err != nil { panic(err) } @@ -139,7 +139,7 @@ func (c *Cluster) startStorage(ctx context.Context) error { // Persist the returned etcd configuration. We decide if we're doing leader election for embedded controllers // based on what the kine wrapper tells us about the datastore. Single-node datastores like sqlite don't require // leader election, while basically all others (etcd, external database, etc) do since they allow multiple servers. - c.etcdConfig = etcdConfig + c.EtcdConfig = etcdConfig c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",") c.config.NoLeaderElect = !etcdConfig.LeaderElect diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 6cb617591f..55196f8387 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -10,23 +10,25 @@ import ( "strings" "github.com/k3s-io/kine/pkg/client" + "github.com/k3s-io/kine/pkg/endpoint" "github.com/rancher/k3s/pkg/bootstrap" "github.com/rancher/k3s/pkg/clientaccess" + "github.com/rancher/k3s/pkg/daemons/config" "github.com/sirupsen/logrus" ) -// save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete +// Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token. // This is used when bootstrapping a cluster from a managed database or external etcd cluster. // This is NOT used with embedded etcd, which bootstraps over HTTP. -func (c *Cluster) save(ctx context.Context, override bool) error { +func Save(ctx context.Context, config *config.Control, etcdConfig endpoint.ETCDConfig, override bool) error { buf := &bytes.Buffer{} - if err := bootstrap.ReadFromDisk(buf, &c.runtime.ControlRuntimeBootstrap); err != nil { + if err := bootstrap.ReadFromDisk(buf, &config.Runtime.ControlRuntimeBootstrap); err != nil { return err } - token := c.config.Token + token := config.Token if token == "" { - tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir) + tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir) if err != nil { return err } @@ -42,12 +44,12 @@ func (c *Cluster) save(ctx context.Context, override bool) error { return err } - storageClient, err := client.New(c.etcdConfig) + storageClient, err := client.New(etcdConfig) if err != nil { return err } - if _, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil { + if _, _, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil { return err } @@ -55,7 +57,7 @@ func (c *Cluster) save(ctx context.Context, override bool) error { if err.Error() == "key exists" { logrus.Warn("bootstrap key already exists") if override { - bsd, err := c.bootstrapKeyData(ctx, storageClient) + bsd, err := bootstrapKeyData(ctx, storageClient) if err != nil { return err } @@ -74,7 +76,7 @@ func (c *Cluster) save(ctx context.Context, override bool) error { // bootstrapKeyData lists keys stored in the datastore with the prefix "/bootstrap", and // will return the first such key. It will return an error if not exactly one key is found. -func (c *Cluster) bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) { +func bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) { bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0) if err != nil { return nil, err @@ -96,7 +98,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { return err } - storageClient, err := client.New(c.etcdConfig) + storageClient, err := client.New(c.EtcdConfig) if err != nil { return err } @@ -119,7 +121,8 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { return err } - value, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) + value, saveBootstrap, err := getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) + c.saveBootstrap = saveBootstrap if err != nil { return err } @@ -139,38 +142,37 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { // hashed with empty string and will check for any key that is hashed by different token than the one // passed to it, it will return error if it finds a key that is hashed with different token and will return // value if it finds the key hashed by passed token or empty string -func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, error) { +func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, bool, error) { emptyStringKey := storageKey("") tokenKey := storageKey(normalizedToken) bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0) if err != nil { - return nil, err + return nil, false, err } if len(bootstrapList) == 0 { - c.saveBootstrap = true - return nil, nil + return nil, true, nil } if len(bootstrapList) > 1 { logrus.Warn("found multiple bootstrap keys in storage") } // check for empty string key and for old token format with k10 prefix - if err := c.migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil { - return nil, err + if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil { + return nil, false, err } // getting the list of bootstrap again after migrating the empty key bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0) if err != nil { - return nil, err + return nil, false, err } for _, bootstrapKV := range bootstrapList { // ensure bootstrap is stored in the current token's key if string(bootstrapKV.Key) == tokenKey { - return &bootstrapKV, nil + return &bootstrapKV, false, nil } } - return nil, errors.New("bootstrap data already found and encrypted with different token") + return nil, false, errors.New("bootstrap data already found and encrypted with different token") } // readTokenFromFile will attempt to get the token from /token if it the file not found @@ -209,7 +211,7 @@ func normalizeToken(token string) (string, error) { // migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and keys that is hashed with old token format before normalizing // then migrate those and resave only with the normalized token -func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { +func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { oldTokenKey := storageKey(oldToken) for _, bootstrapKV := range bootstrapList { diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 7a334b3b2a..2994d6514c 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -155,6 +155,8 @@ type Control struct { ClusterReset bool ClusterResetRestorePath string EncryptSecrets bool + EncryptForce bool + EncryptSkip bool TLSMinVersion uint16 TLSCipherSuites []uint16 EtcdSnapshotName string @@ -197,6 +199,7 @@ type ControlRuntimeBootstrap struct { RequestHeaderCAKey string IPSECKey string EncryptionConfig string + EncryptionHash string } type ControlRuntime struct { @@ -253,7 +256,8 @@ type ControlRuntime struct { ClientETCDCert string ClientETCDKey string - Core *core.Factory + Core *core.Factory + EtcdConfig endpoint.ETCDConfig } type ArgString []string diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index 347a13385c..cc5e332243 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -3,8 +3,10 @@ package deps import ( "crypto" cryptorand "crypto/rand" + "crypto/sha256" "crypto/x509" b64 "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "io/ioutil" @@ -147,6 +149,7 @@ func CreateRuntimeCertFiles(config *config.Control, runtime *config.ControlRunti if config.EncryptSecrets { runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json") + runtime.EncryptionHash = filepath.Join(config.DataDir, "cred", "encryption-state.json") } } @@ -169,7 +172,7 @@ func GenServerDeps(config *config.Control, runtime *config.ControlRuntime) error return err } - if err := genEncryptionConfig(config, runtime); err != nil { + if err := genEncryptionConfigAndState(config, runtime); err != nil { return err } @@ -649,7 +652,7 @@ func expired(certFile string, pool *x509.CertPool) bool { return certutil.IsCertExpired(certificates[0], config.CertificateRenewDays) } -func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlRuntime) error { +func genEncryptionConfigAndState(controlConfig *config.Control, runtime *config.ControlRuntime) error { if !controlConfig.EncryptSecrets { return nil } @@ -690,9 +693,14 @@ func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlR }, }, } - jsonfile, err := json.Marshal(encConfig) + b, err := json.Marshal(encConfig) if err != nil { return err } - return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600) + if err := ioutil.WriteFile(runtime.EncryptionConfig, b, 0600); err != nil { + return err + } + encryptionConfigHash := sha256.Sum256(b) + ann := "start-" + hex.EncodeToString(encryptionConfigHash[:]) + return ioutil.WriteFile(controlConfig.Runtime.EncryptionHash, []byte(ann), 0600) } diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 5db40e3491..21443d3310 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -260,6 +260,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro } runtime.ETCDReady = ready + runtime.EtcdConfig = cluster.EtcdConfig return nil } diff --git a/pkg/deploy/controller.go b/pkg/deploy/controller.go index 06d31c2fbd..5a3dd47d36 100644 --- a/pkg/deploy/controller.go +++ b/pkg/deploy/controller.go @@ -18,10 +18,10 @@ import ( "github.com/rancher/k3s/pkg/agent/util" apisv1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1" controllersv1 "github.com/rancher/k3s/pkg/generated/controllers/k3s.cattle.io/v1" + pkgutil "github.com/rancher/k3s/pkg/util" "github.com/rancher/wrangler/pkg/apply" "github.com/rancher/wrangler/pkg/merr" "github.com/rancher/wrangler/pkg/objectset" - "github.com/rancher/wrangler/pkg/schemes" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -31,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" yamlDecoder "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes" - typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" ) @@ -74,12 +73,7 @@ type watcher struct { // start calls listFiles at regular intervals to trigger application of manifests that have changed on disk. func (w *watcher) start(ctx context.Context, client kubernetes.Interface) { - nodeName := os.Getenv("NODE_NAME") - broadcaster := record.NewBroadcaster() - broadcaster.StartLogging(logrus.Infof) - broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(metav1.NamespaceSystem)}) - w.recorder = broadcaster.NewRecorder(schemes.All, corev1.EventSource{Component: ControllerName, Host: nodeName}) - + w.recorder = pkgutil.BuildControllerEventRecorder(client, ControllerName) force := true for { if err := w.listFiles(force); err == nil { diff --git a/pkg/secretsencrypt/config.go b/pkg/secretsencrypt/config.go new file mode 100644 index 0000000000..1058db06cb --- /dev/null +++ b/pkg/secretsencrypt/config.go @@ -0,0 +1,174 @@ +package secretsencrypt + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/version" + corev1 "k8s.io/api/core/v1" + + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" +) + +const ( + EncryptionStart string = "start" + EncryptionPrepare string = "prepare" + EncryptionRotate string = "rotate" + EncryptionReencryptRequest string = "reencrypt_request" + EncryptionReencryptActive string = "reencrypt_active" + EncryptionReencryptFinished string = "reencrypt_finished" +) + +var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash" + +func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1.ProviderConfiguration, error) { + curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig) + if err != nil { + return nil, err + } + + curEncryption := apiserverconfigv1.EncryptionConfiguration{} + if err = json.Unmarshal(curEncryptionByte, &curEncryption); err != nil { + return nil, err + } + return curEncryption.Resources[0].Providers, nil +} + +func GetEncryptionKeys(runtime *config.ControlRuntime) ([]apiserverconfigv1.Key, error) { + + providers, err := GetEncryptionProviders(runtime) + if err != nil { + return nil, err + } + if len(providers) > 2 { + return nil, fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers)) + } + + var curKeys []apiserverconfigv1.Key + for _, p := range providers { + if p.AESCBC != nil { + curKeys = append(curKeys, p.AESCBC.Keys...) + } + if p.AESGCM != nil || p.KMS != nil || p.Secretbox != nil { + return nil, fmt.Errorf("non-standard encryption keys found") + } + } + return curKeys, nil +} + +func WriteEncryptionConfig(runtime *config.ControlRuntime, keys []apiserverconfigv1.Key, enable bool) error { + + // Placing the identity provider first disables encryption + var providers []apiserverconfigv1.ProviderConfiguration + if enable { + providers = []apiserverconfigv1.ProviderConfiguration{ + { + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: keys, + }, + }, + { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }, + } + } else { + providers = []apiserverconfigv1.ProviderConfiguration{ + { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }, + { + AESCBC: &apiserverconfigv1.AESConfiguration{ + Keys: keys, + }, + }, + } + } + + encConfig := apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{ + { + Resources: []string{"secrets"}, + Providers: providers, + }, + }, + } + jsonfile, err := json.Marshal(encConfig) + if err != nil { + return err + } + return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600) +} + +func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) { + curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig) + if err != nil { + return "", err + } + encryptionConfigHash := sha256.Sum256(curEncryptionByte) + return hex.EncodeToString(encryptionConfigHash[:]), nil +} + +// GenReencryptHash generates a sha256 hash fom the existing secrets keys and +// a new key based on the input arguments. +func GenReencryptHash(runtime *config.ControlRuntime, keyName string) (string, error) { + + keys, err := GetEncryptionKeys(runtime) + if err != nil { + return "", err + } + newKey := apiserverconfigv1.Key{ + Name: keyName, + Secret: "12345", + } + keys = append(keys, newKey) + b, err := json.Marshal(keys) + if err != nil { + return "", err + } + hash := sha256.Sum256(b) + return hex.EncodeToString(hash[:]), nil +} + +func getEncryptionHashFile(runtime *config.ControlRuntime) (string, error) { + curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionHash) + if err != nil { + return "", err + } + return string(curEncryptionByte), nil +} + +func BootstrapEncryptionHashAnnotation(node *corev1.Node, runtime *config.ControlRuntime) error { + existingAnn, err := getEncryptionHashFile(runtime) + if err != nil { + return err + } + node.Annotations[EncryptionHashAnnotation] = existingAnn + return nil +} + +func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, stage string) error { + encryptionConfigHash, err := GenEncryptionConfigHash(runtime) + if err != nil { + return err + } + if node.Annotations == nil { + return fmt.Errorf("node annotations do not exist for %s", node.ObjectMeta.Name) + } + ann := stage + "-" + encryptionConfigHash + node.Annotations[EncryptionHashAnnotation] = ann + if _, err = runtime.Core.Core().V1().Node().Update(node); err != nil { + return err + } + logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name) + return ioutil.WriteFile(runtime.EncryptionHash, []byte(ann), 0600) +} diff --git a/pkg/secretsencrypt/controller.go b/pkg/secretsencrypt/controller.go new file mode 100644 index 0000000000..422d160f7e --- /dev/null +++ b/pkg/secretsencrypt/controller.go @@ -0,0 +1,196 @@ +package secretsencrypt + +import ( + "context" + "fmt" + "strings" + + "github.com/rancher/k3s/pkg/cluster" + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/util" + coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/pager" + "k8s.io/client-go/tools/record" +) + +const ( + controllerAgentName string = "reencrypt-controller" + secretsUpdateStartEvent string = "SecretsUpdateStart" + secretsProgressEvent string = "SecretsProgress" + secretsUpdateCompleteEvent string = "SecretsUpdateComplete" + secretsUpdateErrorEvent string = "SecretsUpdateError" +) + +type handler struct { + ctx context.Context + controlConfig *config.Control + nodes coreclient.NodeController + secrets coreclient.SecretController + recorder record.EventRecorder +} + +func Register( + ctx context.Context, + k8s kubernetes.Interface, + controlConfig *config.Control, + nodes coreclient.NodeController, + secrets coreclient.SecretController, +) error { + h := &handler{ + ctx: ctx, + controlConfig: controlConfig, + nodes: nodes, + secrets: secrets, + recorder: util.BuildControllerEventRecorder(k8s, controllerAgentName), + } + + nodes.OnChange(ctx, "reencrypt-controller", h.onChangeNode) + return nil +} + +// onChangeNode handles changes to Nodes. We are looking for a specific annotation change +func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, error) { + if node == nil { + return nil, nil + } + + ann, ok := node.Annotations[EncryptionHashAnnotation] + if !ok { + return node, nil + } + + if valid, err := h.validateReencryptStage(node, ann); err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } else if !valid { + return node, nil + } + + reencryptHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive) + if err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } + ann = EncryptionReencryptActive + "-" + reencryptHash + node.Annotations[EncryptionHashAnnotation] = ann + node, err = h.nodes.Update(node) + if err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } + + if err := h.updateSecrets(node); err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } + + // If skipping, revert back to the previous stage + if h.controlConfig.EncryptSkip { + BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime) + if node, err := h.nodes.Update(node); err != nil { + return node, err + } + return node, nil + } + + // Remove last key + curKeys, err := GetEncryptionKeys(h.controlConfig.Runtime) + if err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } + + curKeys = curKeys[:len(curKeys)-1] + if err = WriteEncryptionConfig(h.controlConfig.Runtime, curKeys, true); err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } + logrus.Infoln("Removed key: ", curKeys[len(curKeys)-1]) + if err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } + if err := WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished); err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } + if err := cluster.Save(h.ctx, h.controlConfig, h.controlConfig.Runtime.EtcdConfig, true); err != nil { + h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) + return node, err + } + return node, nil +} + +// validateReencryptStage ensures that the request for reencryption is valid and +// that there is only one active reencryption at a time +func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (bool, error) { + + split := strings.Split(annotation, "-") + if len(split) != 2 { + err := fmt.Errorf("invalid annotation %s found on node %s", annotation, node.ObjectMeta.Name) + return false, err + } + stage := split[0] + hash := split[1] + + // Validate the specific stage and the request via sha256 hash + if stage != EncryptionReencryptRequest { + return false, nil + } + if reencryptRequestHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptRequest); err != nil { + return false, err + } else if reencryptRequestHash != hash { + err = fmt.Errorf("invalid hash: %s found on node %s", hash, node.ObjectMeta.Name) + return false, err + } + + nodes, err := h.nodes.List(metav1.ListOptions{}) + if err != nil { + return false, err + } + reencryptActiveHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive) + if err != nil { + return false, err + } + for _, node := range nodes.Items { + if ann, ok := node.Annotations[EncryptionHashAnnotation]; ok { + split := strings.Split(ann, "-") + if len(split) != 2 { + return false, fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name) + } + stage := split[0] + hash := split[1] + if stage == EncryptionReencryptActive && hash == reencryptActiveHash { + return false, fmt.Errorf("another reencrypt is already active") + } + } + } + return true, nil +} + +func (h *handler) updateSecrets(node *corev1.Node) error { + secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { + return h.secrets.List("", opts) + })) + i := 0 + secretPager.EachListItem(h.ctx, metav1.ListOptions{}, func(obj runtime.Object) error { + if secret, ok := obj.(*corev1.Secret); ok { + if _, err := h.secrets.Update(secret); err != nil { + return fmt.Errorf("failed to reencrypted secret: %v", err) + } + if i != 0 && i%10 == 0 { + h.recorder.Eventf(node, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i) + } + i++ + } + return nil + }) + h.recorder.Eventf(node, corev1.EventTypeNormal, secretsUpdateCompleteEvent, "completed reencrypt of %d secrets", i) + return nil +} diff --git a/pkg/server/router.go b/pkg/server/router.go index db45c8895f..ecafb89686 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -48,6 +48,8 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler authed.Path(prefix + "/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA)) authed.Path(prefix + "/config").Handler(configHandler(serverConfig, cfg)) authed.Path(prefix + "/readyz").Handler(readyzHandler(serverConfig)) + authed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig)) + authed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig)) nodeAuthed := mux.NewRouter() nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes")) diff --git a/pkg/server/secrets-encrypt.go b/pkg/server/secrets-encrypt.go new file mode 100644 index 0000000000..ef20254e30 --- /dev/null +++ b/pkg/server/secrets-encrypt.go @@ -0,0 +1,379 @@ +package server + +import ( + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "math/big" + "net/http" + "os" + "strings" + "time" + + "github.com/rancher/k3s/pkg/cluster" + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/secretsencrypt" + "github.com/rancher/wrangler/pkg/generated/controllers/core" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" + "k8s.io/utils/pointer" +) + +const aescbcKeySize = 32 + +type EncryptionState struct { + Stage string `json:"stage"` + ActiveKey string `json:"activekey"` + Enable *bool `json:"enable,omitempty"` + HashMatch bool `json:"hashmatch,omitempty"` + HashError string `json:"hasherror,omitempty"` + InactiveKeys []string `json:"inactivekeys,omitempty"` +} + +type EncryptionRequest struct { + Stage *string `json:"stage,omitempty"` + Enable *bool `json:"enable,omitempty"` + Force bool `json:"force"` + Skip bool `json:"skip"` +} + +func getEncryptionRequest(req *http.Request) (EncryptionRequest, error) { + b, err := ioutil.ReadAll(req.Body) + if err != nil { + return EncryptionRequest{}, err + } + result := EncryptionRequest{} + err = json.Unmarshal(b, &result) + return result, err +} + +func encryptionStatusHandler(server *config.Control) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if req.TLS == nil { + resp.WriteHeader(http.StatusNotFound) + return + } + status, err := encryptionStatus(server) + if err != nil { + genErrorMessage(resp, http.StatusInternalServerError, err) + return + } + b, err := json.Marshal(status) + if err != nil { + genErrorMessage(resp, http.StatusInternalServerError, err) + return + } + resp.Write(b) + }) +} + +func encryptionStatus(server *config.Control) (EncryptionState, error) { + state := EncryptionState{} + providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime) + if os.IsNotExist(err) { + return state, nil + } else if err != nil { + return state, err + } + if providers[1].Identity != nil && providers[0].AESCBC != nil { + state.Enable = pointer.Bool(true) + } else if providers[0].Identity != nil && providers[1].AESCBC != nil || !server.EncryptSecrets { + state.Enable = pointer.Bool(false) + } + + if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), ""); err != nil { + state.HashMatch = false + state.HashError = err.Error() + } else { + state.HashMatch = true + } + stage, _, err := getEncryptionHashAnnotation(server.Runtime.Core.Core()) + if err != nil { + return state, err + } + state.Stage = stage + active := true + for _, p := range providers { + if p.AESCBC != nil { + for _, aesKey := range p.AESCBC.Keys { + if active { + active = false + state.ActiveKey = aesKey.Name + } else { + state.InactiveKeys = append(state.InactiveKeys, aesKey.Name) + } + } + } + if p.Identity != nil { + active = false + } + } + + return state, nil +} + +func encryptionEnable(ctx context.Context, server *config.Control, enable bool) error { + providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime) + if err != nil { + return err + } + if len(providers) > 2 { + return fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers)) + } + curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime) + if err != nil { + return err + } + if providers[1].Identity != nil && providers[0].AESCBC != nil && !enable { + logrus.Infoln("Disabling secrets encryption") + if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil { + return err + } + } else if !enable { + logrus.Infoln("Secrets encryption already disabled") + return nil + } else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable { + logrus.Infoln("Enabling secrets encryption") + if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil { + return err + } + } else if enable { + logrus.Infoln("Secrets encryption already enabled") + return nil + } else { + return fmt.Errorf("unable to enable/disable secrets encryption, unknown configuration") + } + return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true) +} + +func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if req.TLS == nil { + resp.WriteHeader(http.StatusNotFound) + return + } + if req.Method != http.MethodPut { + resp.WriteHeader(http.StatusBadRequest) + return + } + encryptReq, err := getEncryptionRequest(req) + if err != nil { + resp.WriteHeader(http.StatusBadRequest) + resp.Write([]byte(err.Error())) + return + } + if encryptReq.Stage != nil { + switch *encryptReq.Stage { + case secretsencrypt.EncryptionPrepare: + err = encryptionPrepare(ctx, server, encryptReq.Force) + case secretsencrypt.EncryptionRotate: + err = encryptionRotate(ctx, server, encryptReq.Force) + case secretsencrypt.EncryptionReencryptActive: + err = encryptionReencrypt(ctx, server, encryptReq.Force, encryptReq.Skip) + default: + err = fmt.Errorf("unknown stage %s requested", *encryptReq.Stage) + } + } else if encryptReq.Enable != nil { + err = encryptionEnable(ctx, server, *encryptReq.Enable) + } + + if err != nil { + genErrorMessage(resp, http.StatusBadRequest, err) + return + } + resp.WriteHeader(http.StatusOK) + }) +} + +func encryptionPrepare(ctx context.Context, server *config.Control, force bool) error { + states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished + if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil && !force { + return err + } + + curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime) + if err != nil { + return err + } + + if err := AppendNewEncryptionKey(&curKeys); err != nil { + return err + } + logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1]) + + if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil { + return err + } + nodeName := os.Getenv("NODE_NAME") + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + if err = secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare); err != nil { + return err + } + return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true) +} + +func encryptionRotate(ctx context.Context, server *config.Control, force bool) error { + + if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionPrepare); err != nil && !force { + return err + } + + curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime) + if err != nil { + return err + } + + // Right rotate elements + rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...) + + if err = secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true); err != nil { + return err + } + logrus.Infoln("Encryption keys right rotated") + nodeName := os.Getenv("NODE_NAME") + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + if err := secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate); err != nil { + return err + } + return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true) +} + +func encryptionReencrypt(ctx context.Context, server *config.Control, force bool, skip bool) error { + + if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionRotate); err != nil && !force { + return err + } + server.EncryptForce = force + server.EncryptSkip = skip + nodeName := os.Getenv("NODE_NAME") + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest) + if err != nil { + return err + } + ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash + node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann + if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil { + return err + } + logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name) + return nil +} + +func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error { + + aescbcKey := make([]byte, aescbcKeySize) + _, err := rand.Read(aescbcKey) + if err != nil { + return err + } + encodedKey := base64.StdEncoding.EncodeToString(aescbcKey) + + newKey := []apiserverconfigv1.Key{ + { + Name: "aescbckey-" + time.Now().Format(time.RFC3339), + Secret: encodedKey, + }, + } + *keys = append(*keys, newKey...) + return nil +} + +func getEncryptionHashAnnotation(core core.Interface) (string, string, error) { + nodeName := os.Getenv("NODE_NAME") + node, err := core.V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return "", "", err + } + ann := node.Annotations[secretsencrypt.EncryptionHashAnnotation] + split := strings.Split(ann, "-") + if len(split) != 2 { + return "", "", fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name) + } + return split[0], split[1], nil +} + +func getServerNodes(core core.Interface) ([]corev1.Node, error) { + nodes, err := core.V1().Node().List(metav1.ListOptions{}) + if err != nil { + return nil, err + } + var serverNodes []corev1.Node + for _, node := range nodes.Items { + if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" { + serverNodes = append(serverNodes, node) + } + } + return serverNodes, nil +} + +// verifyEncryptionHashAnnotation checks that all nodes are on the same stage, +// and that a request for new stage is valid +func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.Interface, prevStage string) error { + + var firstHash string + var firstNodeName string + first := true + serverNodes, err := getServerNodes(core) + if err != nil { + return err + } + for _, node := range serverNodes { + hash, ok := node.Annotations[secretsencrypt.EncryptionHashAnnotation] + if ok && first { + firstHash = hash + first = false + firstNodeName = node.ObjectMeta.Name + } else if ok && hash != firstHash { + return fmt.Errorf("hash does not match between %s and %s", firstNodeName, node.ObjectMeta.Name) + } + } + + if prevStage == "" { + return nil + } + + oldStage, oldHash, err := getEncryptionHashAnnotation(core) + if err != nil { + return err + } + + encryptionConfigHash, err := secretsencrypt.GenEncryptionConfigHash(runtime) + if err != nil { + return err + } + if !strings.Contains(prevStage, oldStage) { + return fmt.Errorf("incorrect stage: %s found on node %s", oldStage, serverNodes[0].ObjectMeta.Name) + } else if oldHash != encryptionConfigHash { + return fmt.Errorf("invalid hash: %s found on node %s", oldHash, serverNodes[0].ObjectMeta.Name) + } + + return nil +} + +func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error) { + errID, err := rand.Int(rand.Reader, big.NewInt(99999)) + if err != nil { + resp.WriteHeader(http.StatusInternalServerError) + resp.Write([]byte(err.Error())) + return + } + logrus.Warnf("secrets-encrypt-%s: %s", errID.String(), passedErr.Error()) + resp.WriteHeader(statusCode) + resp.Write([]byte(fmt.Sprintf("error secrets-encrypt-%s: see server logs for more info", errID.String()))) +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 5b3dd51fd0..ed8e016fa7 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -25,6 +25,7 @@ import ( "github.com/rancher/k3s/pkg/node" "github.com/rancher/k3s/pkg/nodepassword" "github.com/rancher/k3s/pkg/rootlessports" + "github.com/rancher/k3s/pkg/secretsencrypt" "github.com/rancher/k3s/pkg/servicelb" "github.com/rancher/k3s/pkg/static" "github.com/rancher/k3s/pkg/util" @@ -169,7 +170,7 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err } } - go setControlPlaneRoleLabel(ctx, sc.Core.Core().V1().Node(), config) + go setNodeLabelsAndAnnotations(ctx, sc.Core.Core().V1().Node(), config) go setClusterDNSConfig(ctx, config, sc.Core.Core().V1().ConfigMap()) @@ -232,6 +233,16 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error { return err } + if config.ControlConfig.EncryptSecrets { + if err := secretsencrypt.Register(ctx, + sc.K8s, + &config.ControlConfig, + sc.Core.Core().V1().Node(), + sc.Core.Core().V1().Secret()); err != nil { + return err + } + } + if config.Rootless { return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), @@ -490,7 +501,7 @@ func isSymlink(config string) bool { return false } -func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient, config *Config) error { +func setNodeLabelsAndAnnotations(ctx context.Context, nodes v1.NodeClient, config *Config) error { if config.DisableAgent || config.ControlConfig.DisableAPIServer { return nil } @@ -515,18 +526,25 @@ func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient, config * etcdRoleLabelExists = true } } - if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" && !etcdRoleLabelExists { - break - } if node.Labels == nil { node.Labels = make(map[string]string) } - node.Labels[ControlPlaneRoleLabelKey] = "true" - node.Labels[MasterRoleLabelKey] = "true" + v, ok := node.Labels[ControlPlaneRoleLabelKey] + if !ok || v != "true" || etcdRoleLabelExists { + node.Labels[ControlPlaneRoleLabelKey] = "true" + node.Labels[MasterRoleLabelKey] = "true" + } + + if config.ControlConfig.EncryptSecrets { + if err = secretsencrypt.BootstrapEncryptionHashAnnotation(node, config.ControlConfig.Runtime); err != nil { + logrus.Infof("Unable to set encryption hash annotation %s", err.Error()) + break + } + } _, err = nodes.Update(node) if err == nil { - logrus.Infof("Control-plane role label has been set successfully on node: %s", nodeName) + logrus.Infof("Labels and annotations have been set successfully on node: %s", nodeName) break } select { diff --git a/pkg/util/api.go b/pkg/util/api.go index ebe22a6bd7..33651cd390 100644 --- a/pkg/util/api.go +++ b/pkg/util/api.go @@ -5,15 +5,20 @@ import ( "fmt" "net" "net/http" + "os" "strconv" "time" "github.com/pkg/errors" "github.com/rancher/wrangler/pkg/merr" + "github.com/rancher/wrangler/pkg/schemes" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + coregetter "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" ) // This sets a default duration to wait for the apiserver to become ready. This is primarily used to @@ -72,3 +77,12 @@ func WaitForAPIServerReady(ctx context.Context, client clientset.Interface, time return nil } + +func BuildControllerEventRecorder(k8s clientset.Interface, controllerName string) record.EventRecorder { + logrus.Infof("Creating %s event broadcaster", controllerName) + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(logrus.Infof) + eventBroadcaster.StartRecordingToSink(&coregetter.EventSinkImpl{Interface: k8s.CoreV1().Events(metav1.NamespaceSystem)}) + nodeName := os.Getenv("NODE_NAME") + return eventBroadcaster.NewRecorder(schemes.All, v1.EventSource{Component: controllerName, Host: nodeName}) +} diff --git a/scripts/build b/scripts/build index 3154fd896c..5b4163ce4b 100755 --- a/scripts/build +++ b/scripts/build @@ -78,6 +78,7 @@ rm -f \ bin/containerd-shim-runc-v2 \ bin/k3s-server \ bin/k3s-etcd-snapshot \ + bin/k3s-secrets-encrypt \ bin/k3s-certificate \ bin/kubectl \ bin/crictl \ @@ -108,6 +109,7 @@ CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STAT ln -s containerd ./bin/k3s-agent ln -s containerd ./bin/k3s-server ln -s containerd ./bin/k3s-etcd-snapshot +ln -s containerd ./bin/k3s-secrets-encrypt ln -s containerd ./bin/k3s-certificate ln -s containerd ./bin/kubectl ln -s containerd ./bin/crictl diff --git a/scripts/package-cli b/scripts/package-cli index 60ff38853e..f5638da0aa 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -7,7 +7,7 @@ cd $(dirname $0)/.. GO=${GO-go} -for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-certificate k3s ; do +for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s ; do rm -f bin/$i ln -s containerd bin/$i done diff --git a/tests/integration/localstorage/localstorage_int_test.go b/tests/integration/localstorage/localstorage_int_test.go index c2bdc1b8d4..da1027de26 100644 --- a/tests/integration/localstorage/localstorage_int_test.go +++ b/tests/integration/localstorage/localstorage_int_test.go @@ -15,7 +15,6 @@ import ( var localStorageServer *testutil.K3sServer var localStorageServerArgs = []string{"--cluster-init"} -var testDataDir = "../../testdata/" var _ = BeforeSuite(func() { if !testutil.IsExistingServer() { var err error @@ -37,12 +36,12 @@ var _ = Describe("local storage", func() { }, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) }) It("creates a new pvc", func() { - result, err := testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pvc.yaml") + result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pvc.yaml") Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created")) Expect(err).NotTo(HaveOccurred()) }) It("creates a new pod", func() { - Expect(testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pod.yaml")). + Expect(testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pod.yaml")). To(ContainSubstring("pod/volume-test created")) }) It("shows storage up in kubectl", func() { @@ -51,7 +50,7 @@ var _ = Describe("local storage", func() { }, "45s", "1s").Should(MatchRegexp(`local-path-pvc.+Bound`)) Eventually(func() (string, error) { return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv") - }, "10s", "1s").Should(MatchRegexp(`pvc.+2Gi.+Bound`)) + }, "10s", "1s").Should(MatchRegexp(`pvc.+1Gi.+Bound`)) Eventually(func() (string, error) { return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pod") }, "10s", "1s").Should(MatchRegexp(`volume-test.+Running`)) diff --git a/tests/testdata/localstorage_pod.yaml b/tests/integration/localstorage/testdata/localstorage_pod.yaml similarity index 100% rename from tests/testdata/localstorage_pod.yaml rename to tests/integration/localstorage/testdata/localstorage_pod.yaml diff --git a/tests/testdata/localstorage_pvc.yaml b/tests/integration/localstorage/testdata/localstorage_pvc.yaml similarity index 91% rename from tests/testdata/localstorage_pvc.yaml rename to tests/integration/localstorage/testdata/localstorage_pvc.yaml index 2ad01dc4b0..8018645ab4 100644 --- a/tests/testdata/localstorage_pvc.yaml +++ b/tests/integration/localstorage/testdata/localstorage_pvc.yaml @@ -9,4 +9,4 @@ spec: storageClassName: local-path resources: requests: - storage: 2Gi + storage: 1Gi diff --git a/tests/integration/secretsencryption/secretsencryption_int_test.go b/tests/integration/secretsencryption/secretsencryption_int_test.go new file mode 100644 index 0000000000..66b6e956bd --- /dev/null +++ b/tests/integration/secretsencryption/secretsencryption_int_test.go @@ -0,0 +1,155 @@ +package integration + +import ( + "fmt" + "os" + "regexp" + "testing" + "time" + + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + . "github.com/onsi/gomega" + testutil "github.com/rancher/k3s/tests/util" +) + +var secretsEncryptionServer *testutil.K3sServer +var secretsEncryptionDataDir = "/tmp/k3sse" + +var secretsEncryptionServerArgs = []string{"--secrets-encryption", "-d", secretsEncryptionDataDir} +var _ = BeforeSuite(func() { + if !testutil.IsExistingServer() { + var err error + Expect(os.MkdirAll(secretsEncryptionDataDir, 0777)).To(Succeed()) + secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) + Expect(err).ToNot(HaveOccurred()) + } +}) + +var _ = Describe("secrets encryption rotation", func() { + BeforeEach(func() { + if testutil.IsExistingServer() { + Skip("Test does not support running on existing k3s servers") + } + }) + When("A server starts with secrets encryption", func() { + It("starts up with no problems", func() { + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "get", "pods", "-A") + }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + }) + It("it creates a encryption key", func() { + result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(ContainSubstring("Encryption Status: Enabled")) + Expect(result).To(ContainSubstring("Current Rotation Stage: start")) + }) + }) + When("A server rotates encryption keys", func() { + It("it prepares to rotate", func() { + Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)). + To(ContainSubstring("prepare completed successfully")) + + result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(ContainSubstring("Current Rotation Stage: prepare")) + reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`) + Expect(err).ToNot(HaveOccurred()) + keys := reg.FindAllString(result, -1) + Expect(keys).To(HaveLen(2)) + Expect(keys[0]).To(ContainSubstring("aescbckey")) + Expect(keys[1]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year()))) + }) + It("restarts the server", func() { + var err error + Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) + secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "get", "pods", "-A") + }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + }) + It("rotates the keys", func() { + Eventually(func() (string, error) { + return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir) + }, "10s", "2s").Should(ContainSubstring("rotate completed successfully")) + + result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(ContainSubstring("Current Rotation Stage: rotate")) + reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`) + Expect(err).ToNot(HaveOccurred()) + keys := reg.FindAllString(result, -1) + Expect(keys).To(HaveLen(2)) + Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year()))) + Expect(keys[1]).To(ContainSubstring("aescbckey")) + }) + It("restarts the server", func() { + var err error + Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) + secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "get", "pods", "-A") + }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + time.Sleep(10 * time.Second) + }) + It("reencrypts the keys", func() { + Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)). + To(ContainSubstring("reencryption started")) + Eventually(func() (string, error) { + return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) + }, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) + result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) + Expect(err).NotTo(HaveOccurred()) + reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`) + Expect(err).ToNot(HaveOccurred()) + keys := reg.FindAllString(result, -1) + Expect(keys).To(HaveLen(1)) + Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year()))) + }) + }) + When("A server disables encryption", func() { + It("it triggers the disable", func() { + Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)). + To(ContainSubstring("secrets-encryption disabled")) + + result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(ContainSubstring("Encryption Status: Disabled")) + }) + It("restarts the server", func() { + var err error + Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) + secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() (string, error) { + return testutil.K3sCmd("kubectl", "get", "pods", "-A") + }, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running")) + time.Sleep(10 * time.Second) + }) + It("reencrypts the keys", func() { + result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(ContainSubstring("reencryption started")) + + result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(ContainSubstring("Encryption Status: Disabled")) + }) + }) +}) + +var _ = AfterSuite(func() { + if !testutil.IsExistingServer() { + Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed()) + Expect(testutil.K3sRemoveDataDir(secretsEncryptionDataDir)).To(Succeed()) + } +}) + +func Test_IntegrationSecretsEncryption(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{ + reporters.NewJUnitReporter("/tmp/results/junit-se.xml"), + }) +} diff --git a/tests/util/cmd.go b/tests/util/cmd.go index a928c46163..e0d7ec3c4a 100644 --- a/tests/util/cmd.go +++ b/tests/util/cmd.go @@ -80,6 +80,18 @@ func K3sCmd(cmdName string, cmdArgs ...string) (string, error) { return string(byteOut), err } +// K3sRemoveDataDir removes the provided directory as root +func K3sRemoveDataDir(dataDir string) error { + var cmd *exec.Cmd + if IsRoot() { + cmd = exec.Command("rm", "-rf", dataDir) + } else { + cmd = exec.Command("sudo", "rm", "-rf", dataDir) + } + _, err := cmd.CombinedOutput() + return err +} + func contains(source []string, target string) bool { for _, s := range source { if s == target { @@ -168,11 +180,5 @@ func K3sKillServer(server *K3sServer) error { return err } } - if err := flock.Release(server.lock); err != nil { - return err - } - if !flock.CheckLock(lockFile) { - return os.RemoveAll(lockFile) - } - return nil + return flock.Release(server.lock) }