Mirror/k3s
1
0
Fork 0
mirror of https://github.com/k3s-io/k3s.git synced 2024-06-07 19:41:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Secrets-encryption rotation (#4372)

Browse Source 
* Regular CLI framework for encrypt commands
* New secrets-encryption feature
* New integration test
* fixes for flaky integration test CI
* Fix to bootstrap on restart of existing nodes
* Consolidate event recorder

Signed-off-by: Derek Nola <derek.nola@suse.com>
This commit is contained in:
Derek Nola 2021-12-07 14:31:32 -08:00 committed by GitHub
parent 7d3447ceff
commit bcb662926d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1460 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/integration.yaml vendored
View File

@ -56,17 +56,8 @@ jobs:
- name: Run Integration Tests
run: |
chmod +x ./dist/artifacts/k3s
go test -coverpkg=./... -coverprofile=coverage.out ./pkg/... -run Integration
go tool cover -func coverage.out
# these tests do not relate to coverage and must be run separately
go test ./tests/integration/... -run Integration
go test ./pkg/... ./tests/integration/... -run Integration
- name: On Failure, Launch Debug Session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
timeout-minutes: 5
- name: Upload Results To Codecov
uses: codecov/codecov-action@v1
with:
files: ./coverage.out
flags: inttests # optional
verbose: true # optional (default = false)
timeout-minutes: 5

32
cmd/encrypt/main.go Normal file
View File

@ -0,0 +1,32 @@
package main
import (
"context"
"errors"
"os"
"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/rancher/k3s/pkg/cli/secretsencrypt"
"github.com/rancher/k3s/pkg/configfilearg"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
func main() {
app := cmds.NewApp()
app.Commands = []cli.Command{
cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
cmds.NewSecretsEncryptSubcommands(
secretsencrypt.Status,
secretsencrypt.Enable,
secretsencrypt.Disable,
secretsencrypt.Prepare,
secretsencrypt.Rotate,
secretsencrypt.Reencrypt),
),
}
if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
logrus.Fatal(err)
}
}

10
cmd/k3s/main.go
View File

@ -35,6 +35,7 @@ func main() {
}
etcdsnapshotCommand := internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)
secretsencryptCommand := internalCLIAction(version.Program+"-"+cmds.SecretsEncryptCommand, dataDir, os.Args)
certCommand := internalCLIAction(version.Program+"-"+cmds.CertCommand, dataDir, os.Args)
// Handle subcommand invocation (k3s server, k3s crictl, etc)
@ -53,6 +54,15 @@ func main() {
etcdsnapshotCommand,
etcdsnapshotCommand),
),
cmds.NewSecretsEncryptCommand(secretsencryptCommand,
cmds.NewSecretsEncryptSubcommands(
secretsencryptCommand,
secretsencryptCommand,
secretsencryptCommand,
secretsencryptCommand,
secretsencryptCommand,
secretsencryptCommand),
),
cmds.NewCertCommand(
cmds.NewCertSubcommands(
certCommand),

10
cmd/server/main.go
View File

@ -15,6 +15,7 @@ import (
"github.com/rancher/k3s/pkg/cli/ctr"
"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
"github.com/rancher/k3s/pkg/cli/kubectl"
"github.com/rancher/k3s/pkg/cli/secretsencrypt"
"github.com/rancher/k3s/pkg/cli/server"
"github.com/rancher/k3s/pkg/configfilearg"
"github.com/rancher/k3s/pkg/containerd"
@ -53,6 +54,15 @@ func main() {
etcdsnapshot.Prune,
etcdsnapshot.Run),
),
cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
cmds.NewSecretsEncryptSubcommands(
secretsencrypt.Status,
secretsencrypt.Enable,
secretsencrypt.Disable,
secretsencrypt.Prepare,
secretsencrypt.Rotate,
secretsencrypt.Reencrypt),
),
cmds.NewCertCommand(
cmds.NewCertSubcommands(
cert.Run),

10
main.go
View File

@ -17,6 +17,7 @@ import (
"github.com/rancher/k3s/pkg/cli/crictl"
"github.com/rancher/k3s/pkg/cli/etcdsnapshot"
"github.com/rancher/k3s/pkg/cli/kubectl"
"github.com/rancher/k3s/pkg/cli/secretsencrypt"
"github.com/rancher/k3s/pkg/cli/server"
"github.com/rancher/k3s/pkg/configfilearg"
"github.com/sirupsen/logrus"
@ -37,6 +38,15 @@ func main() {
etcdsnapshot.Prune,
etcdsnapshot.Run),
),
cmds.NewSecretsEncryptCommand(cli.ShowAppHelp,
cmds.NewSecretsEncryptSubcommands(
secretsencrypt.Status,
secretsencrypt.Enable,
secretsencrypt.Disable,
secretsencrypt.Prepare,
secretsencrypt.Rotate,
secretsencrypt.Reencrypt),
),
cmds.NewCertCommand(
cmds.NewCertSubcommands(
cert.Run),

19
pkg/cli/cmds/agent.go
View File

@ -56,9 +56,15 @@ type AgentShared struct {
}
var (
appName = filepath.Base(os.Args[0])
AgentConfig Agent
NodeIPFlag = cli.StringSliceFlag{
appName = filepath.Base(os.Args[0])
AgentConfig Agent
AgentTokenFlag = cli.StringFlag{
Name: "token,t",
Usage: "(cluster) Token to use for authentication",
EnvVar: version.ProgramUpper + "_TOKEN",
Destination: &AgentConfig.Token,
}
NodeIPFlag = cli.StringSliceFlag{
Name: "node-ip,i",
Usage: "(agent/networking) IPv4/IPv6 addresses to advertise for node",
Value: &AgentConfig.NodeIP,
@ -217,12 +223,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
VModule,
LogFile,
AlsoLogToStderr,
cli.StringFlag{
Name: "token,t",
Usage: "(cluster) Token to use for authentication",
EnvVar: version.ProgramUpper + "_TOKEN",
Destination: &AgentConfig.Token,
},
AgentTokenFlag,
cli.StringFlag{
Name: "token-file",
Usage: "(cluster) Token file to use for authentication",

6
pkg/cli/cmds/etcd_snapshot.go
View File

@ -20,11 +20,7 @@ var EtcdSnapshotFlags = []cli.Flag{
EnvVar: version.ProgramUpper + "_NODE_NAME",
Destination: &AgentConfig.NodeName,
},
cli.StringFlag{
Name: "data-dir,d",
Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
Destination: &ServerConfig.DataDir,
},
DataDirFlag,
&cli.StringFlag{
Name: "dir,etcd-snapshot-dir",
Usage: "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",

94
pkg/cli/cmds/secrets_encrypt.go Normal file
View File

@ -0,0 +1,94 @@
package cmds
import (
"github.com/urfave/cli"
)
const SecretsEncryptCommand = "secrets-encrypt"
var EncryptFlags = []cli.Flag{
DataDirFlag,
ServerToken,
}
func NewSecretsEncryptCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command {
return cli.Command{
Name: SecretsEncryptCommand,
Usage: "Control secrets encryption and keys rotation",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: action,
Subcommands: subcommands,
}
}
func NewSecretsEncryptSubcommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) []cli.Command {
return []cli.Command{
{
Name: "status",
Usage: "Print current status of secrets encryption",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: status,
Flags: EncryptFlags,
},
{
Name: "enable",
Usage: "Enable secrets encryption",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: enable,
Flags: EncryptFlags,
},
{
Name: "disable",
Usage: "Disable secrets encryption",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: disable,
Flags: EncryptFlags,
},
{
Name: "prepare",
Usage: "Prepare for encryption keys rotation",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: prepare,
Flags: append(EncryptFlags, &cli.BoolFlag{
Name: "f,force",
Usage: "Force preparation.",
Destination: &ServerConfig.EncryptForce,
}),
},
{
Name: "rotate",
Usage: "Rotate secrets encryption keys",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: rotate,
Flags: append(EncryptFlags, &cli.BoolFlag{
Name: "f,force",
Usage: "Force key rotation.",
Destination: &ServerConfig.EncryptForce,
}),
},
{
Name: "reencrypt",
Usage: "Reencrypt all data with new encryption key",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: reencrypt,
Flags: append(EncryptFlags,
&cli.BoolFlag{
Name: "f,force",
Usage: "Force secrets reencryption.",
Destination: &ServerConfig.EncryptForce,
},
&cli.BoolFlag{
Name: "skip",
Usage: "Skip removing old key",
Destination: &ServerConfig.EncryptSkip,
}),
},
}
}

28
pkg/cli/cmds/server.go
View File

@ -74,6 +74,8 @@ type Server struct {
ClusterReset bool
ClusterResetRestorePath string
EncryptSecrets bool
EncryptForce bool
EncryptSkip bool
SystemDefaultRegistry string
StartupHooks []StartupHook
EtcdSnapshotName string
@ -97,7 +99,18 @@ type Server struct {
var (
ServerConfig Server
ClusterCIDR = cli.StringSliceFlag{
DataDirFlag = cli.StringFlag{
Name: "data-dir,d",
Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
Destination: &ServerConfig.DataDir,
}
ServerToken = cli.StringFlag{
Name: "token,t",
Usage: "(cluster) Shared secret used to join a server or agent to a cluster",
Destination: &ServerConfig.Token,
EnvVar: version.ProgramUpper + "_TOKEN",
}
ClusterCIDR = cli.StringSliceFlag{
Name: "cluster-cidr",
Usage: "(networking) IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16)",
Value: &ServerConfig.ClusterCIDR,
@ -179,11 +192,7 @@ var ServerFlags = []cli.Flag{
Usage: "(listener) Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert",
Value: &ServerConfig.TLSSan,
},
cli.StringFlag{
Name: "data-dir,d",
Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
Destination: &ServerConfig.DataDir,
},
DataDirFlag,
ClusterCIDR,
ServiceCIDR,
ServiceNodePortRange,
@ -195,12 +204,7 @@ var ServerFlags = []cli.Flag{
Destination: &ServerConfig.FlannelBackend,
Value: "vxlan",
},
cli.StringFlag{
Name: "token,t",
Usage: "(cluster) Shared secret used to join a server or agent to a cluster",
Destination: &ServerConfig.Token,
EnvVar: version.ProgramUpper + "_TOKEN",
},
ServerToken,
cli.StringFlag{
Name: "token-file",
Usage: "(cluster) File containing the cluster-secret/token",

216
pkg/cli/secretsencrypt/secrets_encrypt.go Normal file
View File

@ -0,0 +1,216 @@
package secretsencrypt
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"text/tabwriter"
"github.com/erikdubbelboer/gspt"
"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/secretsencrypt"
"github.com/rancher/k3s/pkg/server"
"github.com/rancher/k3s/pkg/version"
"github.com/urfave/cli"
"k8s.io/utils/pointer"
)
func commandPrep(app *cli.Context, cfg *cmds.Server) (config.Control, *clientaccess.Info, error) {
var controlConfig config.Control
var err error
// hide process arguments from ps output, since they may contain
// database credentials or other secrets.
gspt.SetProcTitle(os.Args[0] + " encrypt")
controlConfig.DataDir, err = server.ResolveDataDir(cfg.DataDir)
if err != nil {
return controlConfig, nil, err
}
if cfg.ServerURL == "" {
cfg.ServerURL = "https://127.0.0.1:6443"
}
if cfg.Token == "" {
fp := filepath.Join(controlConfig.DataDir, "token")
tokenByte, err := ioutil.ReadFile(fp)
if err != nil {
return controlConfig, nil, err
}
controlConfig.Token = string(bytes.TrimRight(tokenByte, "\n"))
} else {
controlConfig.Token = cfg.Token
}
controlConfig.EncryptForce = cfg.EncryptForce
controlConfig.EncryptSkip = cfg.EncryptSkip
info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, controlConfig.Token, "node")
if err != nil {
return controlConfig, nil, err
}
return controlConfig, info, nil
}
func Enable(app *cli.Context) error {
var err error
if err = cmds.InitLogging(); err != nil {
return err
}
_, info, err := commandPrep(app, &cmds.ServerConfig)
if err != nil {
return err
}
b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(true)})
if err != nil {
return err
}
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
return err
}
fmt.Println("secrets-encryption enabled")
return nil
}
func Disable(app *cli.Context) error {
if err := cmds.InitLogging(); err != nil {
return err
}
_, info, err := commandPrep(app, &cmds.ServerConfig)
if err != nil {
return err
}
b, err := json.Marshal(server.EncryptionRequest{Enable: pointer.Bool(false)})
if err != nil {
return err
}
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
return err
}
fmt.Println("secrets-encryption disabled")
return nil
}
func Status(app *cli.Context) error {
if err := cmds.InitLogging(); err != nil {
return err
}
_, info, err := commandPrep(app, &cmds.ServerConfig)
if err != nil {
return err
}
data, err := info.Get("/v1-" + version.Program + "/encrypt/status")
if err != nil {
return err
}
status := server.EncryptionState{}
if err := json.Unmarshal(data, &status); err != nil {
return err
}
if status.Enable == nil {
fmt.Println("Encryption Status: Disabled, no configuration file found")
return nil
}
var statusOutput string
if *status.Enable {
statusOutput += "Encryption Status: Enabled\n"
} else {
statusOutput += "Encryption Status: Disabled\n"
}
statusOutput += fmt.Sprintln("Current Rotation Stage:", status.Stage)
if status.HashMatch {
statusOutput += fmt.Sprintln("Server Encryption Hashes: All hashes match")
} else {
statusOutput += fmt.Sprintf("Server Encryption Hashes: %s\n", status.HashError)
}
var tabBuffer bytes.Buffer
w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0)
fmt.Fprintf(w, "\n")
fmt.Fprintf(w, "Active\tKey Type\tName\n")
fmt.Fprintf(w, "------\t--------\t----\n")
if status.ActiveKey != "" {
fmt.Fprintf(w, " *\t%s\t%s\n", "AES-CBC", status.ActiveKey)
}
for _, k := range status.InactiveKeys {
fmt.Fprintf(w, "\t%s\t%s\n", "AES-CBC", k)
}
w.Flush()
fmt.Println(statusOutput + tabBuffer.String())
return nil
}
func Prepare(app *cli.Context) error {
var err error
if err = cmds.InitLogging(); err != nil {
return err
}
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
if err != nil {
return err
}
b, err := json.Marshal(server.EncryptionRequest{
Stage: pointer.StringPtr(secretsencrypt.EncryptionPrepare),
Force: controlConfig.EncryptForce,
})
if err != nil {
return err
}
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
return err
}
fmt.Println("prepare completed successfully")
return nil
}
func Rotate(app *cli.Context) error {
if err := cmds.InitLogging(); err != nil {
return err
}
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
if err != nil {
return err
}
b, err := json.Marshal(server.EncryptionRequest{
Stage: pointer.StringPtr(secretsencrypt.EncryptionRotate),
Force: controlConfig.EncryptForce,
})
if err != nil {
return err
}
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
return err
}
fmt.Println("rotate completed successfully")
return nil
}
func Reencrypt(app *cli.Context) error {
var err error
if err = cmds.InitLogging(); err != nil {
return err
}
controlConfig, info, err := commandPrep(app, &cmds.ServerConfig)
if err != nil {
return err
}
b, err := json.Marshal(server.EncryptionRequest{
Stage: pointer.StringPtr(secretsencrypt.EncryptionReencryptActive),
Force: controlConfig.EncryptForce,
Skip: controlConfig.EncryptSkip,
})
if err != nil {
return err
}
if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
return err
}
fmt.Println("reencryption started")
return nil
}

37
pkg/clientaccess/token.go
View File

@ -1,6 +1,7 @@
package clientaccess
import (
"bytes"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
@ -186,6 +187,16 @@ func (i *Info) Get(path string) ([]byte, error) {
return get(u.String(), GetHTTPClient(i.CACerts), i.Username, i.Password)
}
// Put makes a request to a subpath of info's BaseURL
func (i *Info) Put(path string, body []byte) error {
u, err := url.Parse(i.BaseURL)
if err != nil {
return err
}
u.Path = path
return put(u.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password)
}
// setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
// and storing the CA bundle.
func (i *Info) setServer(server string) error {
@ -288,6 +299,32 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro
return ioutil.ReadAll(resp.Body)
}
// put makes a request to a url using a provided client, username, and password
// only an error is returned
func put(u string, body []byte, client *http.Client, username, password string) error {
req, err := http.NewRequest(http.MethodPut, u, bytes.NewBuffer(body))
if err != nil {
return err
}
if username != "" {
req.SetBasicAuth(username, password)
}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
respBody, _ := ioutil.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("%s: %s %s", u, resp.Status, string(respBody))
}
return nil
}
func FormatToken(token, certFile string) (string, error) {
if len(token) == 0 {
return token, nil

13
pkg/cluster/bootstrap.go
View File

@ -188,17 +188,18 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
if err != nil {
return false, false, err
}
if isInitialized {
// If the database is initialized we skip bootstrapping; if the user wants to rejoin a
// cluster they need to delete the database.
logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName())
// This is a workaround for an issue that can be caused by terminating the cluster bootstrap before
// etcd is promoted from learner. Odds are we won't need this info, and we don't want to fail startup
// due to failure to retrieve it as this will break cold cluster restart, so we ignore any errors.
if c.config.JoinURL != "" && c.config.Token != "" {
c.clientAccessInfo, _ = clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
logrus.Infof("Joining %s cluster already initialized, forcing reconciliation", c.managedDB.EndpointName())
return true, true, nil
}
// If the database is initialized we skip bootstrapping; if the user wants to rejoin a
// cluster they need to delete the database.
logrus.Infof("Managed %s cluster bootstrap already complete and initialized", c.managedDB.EndpointName())
return false, true, nil
} else if c.config.JoinURL == "" {
// Not initialized, not joining - must be initializing (cluster-init)
@ -353,7 +354,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
if ec != nil {
etcdConfig = *ec
} else {
etcdConfig = c.etcdConfig
etcdConfig = c.EtcdConfig
}
storageClient, err := client.New(etcdConfig)
@ -366,7 +367,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
RETRY:
for {
value, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
value, c.saveBootstrap, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
if err != nil {
if strings.Contains(err.Error(), "not supported for learner") {
for range ticker.C {

4
pkg/cluster/bootstrap_test.go
View File

@ -130,7 +130,7 @@ func TestCluster_certDirsExist(t *testing.T) {
config: tt.fields.config,
runtime: tt.fields.runtime,
managedDB: tt.fields.managedDB,
etcdConfig: tt.fields.etcdConfig,
EtcdConfig: tt.fields.etcdConfig,
storageStarted: tt.fields.storageStarted,
saveBootstrap: tt.fields.saveBootstrap,
}
@ -240,7 +240,7 @@ func TestCluster_Snapshot(t *testing.T) {
config: tt.fields.config,
runtime: tt.fields.runtime,
managedDB: tt.fields.managedDB,
etcdConfig: tt.fields.etcdConfig,
EtcdConfig: tt.fields.etcdConfig,
joining: tt.fields.joining,
storageStarted: tt.fields.storageStarted,
saveBootstrap: tt.fields.saveBootstrap,

8
pkg/cluster/cluster.go
View File

@ -20,7 +20,7 @@ type Cluster struct {
config *config.Control
runtime *config.ControlRuntime
managedDB managed.Driver
etcdConfig endpoint.ETCDConfig
EtcdConfig endpoint.ETCDConfig
joining bool
storageStarted bool
saveBootstrap bool
@ -85,7 +85,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
// if necessary, store bootstrap data to datastore
if c.saveBootstrap {
if err := c.save(ctx, false); err != nil {
if err := Save(ctx, c.config, c.EtcdConfig, false); err != nil {
return nil, err
}
}
@ -99,7 +99,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
for {
select {
case <-ready:
if err := c.save(ctx, false); err != nil {
if err := Save(ctx, c.config, c.EtcdConfig, false); err != nil {
panic(err)
}
@ -139,7 +139,7 @@ func (c *Cluster) startStorage(ctx context.Context) error {
// Persist the returned etcd configuration. We decide if we're doing leader election for embedded controllers
// based on what the kine wrapper tells us about the datastore. Single-node datastores like sqlite don't require
// leader election, while basically all others (etcd, external database, etc) do since they allow multiple servers.
c.etcdConfig = etcdConfig
c.EtcdConfig = etcdConfig
c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig
c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
c.config.NoLeaderElect = !etcdConfig.LeaderElect

44
pkg/cluster/storage.go
View File

@ -10,23 +10,25 @@ import (
"strings"
"github.com/k3s-io/kine/pkg/client"
"github.com/k3s-io/kine/pkg/endpoint"
"github.com/rancher/k3s/pkg/bootstrap"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/sirupsen/logrus"
)
// save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
// Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
// snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token.
// This is used when bootstrapping a cluster from a managed database or external etcd cluster.
// This is NOT used with embedded etcd, which bootstraps over HTTP.
func (c *Cluster) save(ctx context.Context, override bool) error {
func Save(ctx context.Context, config *config.Control, etcdConfig endpoint.ETCDConfig, override bool) error {
buf := &bytes.Buffer{}
if err := bootstrap.ReadFromDisk(buf, &c.runtime.ControlRuntimeBootstrap); err != nil {
if err := bootstrap.ReadFromDisk(buf, &config.Runtime.ControlRuntimeBootstrap); err != nil {
return err
}
token := c.config.Token
token := config.Token
if token == "" {
tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir)
tokenFromFile, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
if err != nil {
return err
}
@ -42,12 +44,12 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
return err
}
storageClient, err := client.New(c.etcdConfig)
storageClient, err := client.New(etcdConfig)
if err != nil {
return err
}
if _, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil {
if _, _, err = getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token); err != nil {
return err
}
@ -55,7 +57,7 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
if err.Error() == "key exists" {
logrus.Warn("bootstrap key already exists")
if override {
bsd, err := c.bootstrapKeyData(ctx, storageClient)
bsd, err := bootstrapKeyData(ctx, storageClient)
if err != nil {
return err
}
@ -74,7 +76,7 @@ func (c *Cluster) save(ctx context.Context, override bool) error {
// bootstrapKeyData lists keys stored in the datastore with the prefix "/bootstrap", and
// will return the first such key. It will return an error if not exactly one key is found.
func (c *Cluster) bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) {
func bootstrapKeyData(ctx context.Context, storageClient client.Client) (*client.Value, error) {
bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
if err != nil {
return nil, err
@ -96,7 +98,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
return err
}
storageClient, err := client.New(c.etcdConfig)
storageClient, err := client.New(c.EtcdConfig)
if err != nil {
return err
}
@ -119,7 +121,8 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
return err
}
value, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
value, saveBootstrap, err := getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
c.saveBootstrap = saveBootstrap
if err != nil {
return err
}
@ -139,38 +142,37 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
// hashed with empty string and will check for any key that is hashed by different token than the one
// passed to it, it will return error if it finds a key that is hashed with different token and will return
// value if it finds the key hashed by passed token or empty string
func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, error) {
func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, bool, error) {
emptyStringKey := storageKey("")
tokenKey := storageKey(normalizedToken)
bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
if err != nil {
return nil, err
return nil, false, err
}
if len(bootstrapList) == 0 {
c.saveBootstrap = true
return nil, nil
return nil, true, nil
}
if len(bootstrapList) > 1 {
logrus.Warn("found multiple bootstrap keys in storage")
}
// check for empty string key and for old token format with k10 prefix
if err := c.migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
return nil, err
if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
return nil, false, err
}
// getting the list of bootstrap again after migrating the empty key
bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0)
if err != nil {
return nil, err
return nil, false, err
}
for _, bootstrapKV := range bootstrapList {
// ensure bootstrap is stored in the current token's key
if string(bootstrapKV.Key) == tokenKey {
return &bootstrapKV, nil
return &bootstrapKV, false, nil
}
}
return nil, errors.New("bootstrap data already found and encrypted with different token")
return nil, false, errors.New("bootstrap data already found and encrypted with different token")
}
// readTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
@ -209,7 +211,7 @@ func normalizeToken(token string) (string, error) {
// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is
// hashed with empty string and keys that is hashed with old token format before normalizing
// then migrate those and resave only with the normalized token
func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
oldTokenKey := storageKey(oldToken)
for _, bootstrapKV := range bootstrapList {

6
pkg/daemons/config/types.go
View File

@ -155,6 +155,8 @@ type Control struct {
ClusterReset bool
ClusterResetRestorePath string
EncryptSecrets bool
EncryptForce bool
EncryptSkip bool
TLSMinVersion uint16
TLSCipherSuites []uint16
EtcdSnapshotName string
@ -197,6 +199,7 @@ type ControlRuntimeBootstrap struct {
RequestHeaderCAKey string
IPSECKey string
EncryptionConfig string
EncryptionHash string
}
type ControlRuntime struct {
@ -253,7 +256,8 @@ type ControlRuntime struct {
ClientETCDCert string
ClientETCDKey string
Core *core.Factory
Core *core.Factory
EtcdConfig endpoint.ETCDConfig
}
type ArgString []string

16
pkg/daemons/control/deps/deps.go
View File

@ -3,8 +3,10 @@ package deps
import (
"crypto"
cryptorand "crypto/rand"
"crypto/sha256"
"crypto/x509"
b64 "encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
@ -147,6 +149,7 @@ func CreateRuntimeCertFiles(config *config.Control, runtime *config.ControlRunti
if config.EncryptSecrets {
runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json")
runtime.EncryptionHash = filepath.Join(config.DataDir, "cred", "encryption-state.json")
}
}
@ -169,7 +172,7 @@ func GenServerDeps(config *config.Control, runtime *config.ControlRuntime) error
return err
}
if err := genEncryptionConfig(config, runtime); err != nil {
if err := genEncryptionConfigAndState(config, runtime); err != nil {
return err
}
@ -649,7 +652,7 @@ func expired(certFile string, pool *x509.CertPool) bool {
return certutil.IsCertExpired(certificates[0], config.CertificateRenewDays)
}
func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlRuntime) error {
func genEncryptionConfigAndState(controlConfig *config.Control, runtime *config.ControlRuntime) error {
if !controlConfig.EncryptSecrets {
return nil
}
@ -690,9 +693,14 @@ func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlR
},
},
}
jsonfile, err := json.Marshal(encConfig)
b, err := json.Marshal(encConfig)
if err != nil {
return err
}
return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
if err := ioutil.WriteFile(runtime.EncryptionConfig, b, 0600); err != nil {
return err
}
encryptionConfigHash := sha256.Sum256(b)
ann := "start-" + hex.EncodeToString(encryptionConfigHash[:])
return ioutil.WriteFile(controlConfig.Runtime.EncryptionHash, []byte(ann), 0600)
}

1
pkg/daemons/control/server.go
View File

@ -260,6 +260,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
}
runtime.ETCDReady = ready
runtime.EtcdConfig = cluster.EtcdConfig
return nil
}

10
pkg/deploy/controller.go
View File

@ -18,10 +18,10 @@ import (
"github.com/rancher/k3s/pkg/agent/util"
apisv1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1"
controllersv1 "github.com/rancher/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
pkgutil "github.com/rancher/k3s/pkg/util"
"github.com/rancher/wrangler/pkg/apply"
"github.com/rancher/wrangler/pkg/merr"
"github.com/rancher/wrangler/pkg/objectset"
"github.com/rancher/wrangler/pkg/schemes"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
@ -31,7 +31,6 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
yamlDecoder "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/kubernetes"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record"
)
@ -74,12 +73,7 @@ type watcher struct {
// start calls listFiles at regular intervals to trigger application of manifests that have changed on disk.
func (w *watcher) start(ctx context.Context, client kubernetes.Interface) {
nodeName := os.Getenv("NODE_NAME")
broadcaster := record.NewBroadcaster()
broadcaster.StartLogging(logrus.Infof)
broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(metav1.NamespaceSystem)})
w.recorder = broadcaster.NewRecorder(schemes.All, corev1.EventSource{Component: ControllerName, Host: nodeName})
w.recorder = pkgutil.BuildControllerEventRecorder(client, ControllerName)
force := true
for {
if err := w.listFiles(force); err == nil {

174
pkg/secretsencrypt/config.go Normal file
View File

@ -0,0 +1,174 @@
package secretsencrypt
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/version"
corev1 "k8s.io/api/core/v1"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
)
const (
EncryptionStart string = "start"
EncryptionPrepare string = "prepare"
EncryptionRotate string = "rotate"
EncryptionReencryptRequest string = "reencrypt_request"
EncryptionReencryptActive string = "reencrypt_active"
EncryptionReencryptFinished string = "reencrypt_finished"
)
var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash"
func GetEncryptionProviders(runtime *config.ControlRuntime) ([]apiserverconfigv1.ProviderConfiguration, error) {
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig)
if err != nil {
return nil, err
}
curEncryption := apiserverconfigv1.EncryptionConfiguration{}
if err = json.Unmarshal(curEncryptionByte, &curEncryption); err != nil {
return nil, err
}
return curEncryption.Resources[0].Providers, nil
}
func GetEncryptionKeys(runtime *config.ControlRuntime) ([]apiserverconfigv1.Key, error) {
providers, err := GetEncryptionProviders(runtime)
if err != nil {
return nil, err
}
if len(providers) > 2 {
return nil, fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
}
var curKeys []apiserverconfigv1.Key
for _, p := range providers {
if p.AESCBC != nil {
curKeys = append(curKeys, p.AESCBC.Keys...)
}
if p.AESGCM != nil || p.KMS != nil || p.Secretbox != nil {
return nil, fmt.Errorf("non-standard encryption keys found")
}
}
return curKeys, nil
}
func WriteEncryptionConfig(runtime *config.ControlRuntime, keys []apiserverconfigv1.Key, enable bool) error {
// Placing the identity provider first disables encryption
var providers []apiserverconfigv1.ProviderConfiguration
if enable {
providers = []apiserverconfigv1.ProviderConfiguration{
{
AESCBC: &apiserverconfigv1.AESConfiguration{
Keys: keys,
},
},
{
Identity: &apiserverconfigv1.IdentityConfiguration{},
},
}
} else {
providers = []apiserverconfigv1.ProviderConfiguration{
{
Identity: &apiserverconfigv1.IdentityConfiguration{},
},
{
AESCBC: &apiserverconfigv1.AESConfiguration{
Keys: keys,
},
},
}
}
encConfig := apiserverconfigv1.EncryptionConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "EncryptionConfiguration",
APIVersion: "apiserver.config.k8s.io/v1",
},
Resources: []apiserverconfigv1.ResourceConfiguration{
{
Resources: []string{"secrets"},
Providers: providers,
},
},
}
jsonfile, err := json.Marshal(encConfig)
if err != nil {
return err
}
return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
}
func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) {
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionConfig)
if err != nil {
return "", err
}
encryptionConfigHash := sha256.Sum256(curEncryptionByte)
return hex.EncodeToString(encryptionConfigHash[:]), nil
}
// GenReencryptHash generates a sha256 hash fom the existing secrets keys and
// a new key based on the input arguments.
func GenReencryptHash(runtime *config.ControlRuntime, keyName string) (string, error) {
keys, err := GetEncryptionKeys(runtime)
if err != nil {
return "", err
}
newKey := apiserverconfigv1.Key{
Name: keyName,
Secret: "12345",
}
keys = append(keys, newKey)
b, err := json.Marshal(keys)
if err != nil {
return "", err
}
hash := sha256.Sum256(b)
return hex.EncodeToString(hash[:]), nil
}
func getEncryptionHashFile(runtime *config.ControlRuntime) (string, error) {
curEncryptionByte, err := ioutil.ReadFile(runtime.EncryptionHash)
if err != nil {
return "", err
}
return string(curEncryptionByte), nil
}
func BootstrapEncryptionHashAnnotation(node *corev1.Node, runtime *config.ControlRuntime) error {
existingAnn, err := getEncryptionHashFile(runtime)
if err != nil {
return err
}
node.Annotations[EncryptionHashAnnotation] = existingAnn
return nil
}
func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, stage string) error {
encryptionConfigHash, err := GenEncryptionConfigHash(runtime)
if err != nil {
return err
}
if node.Annotations == nil {
return fmt.Errorf("node annotations do not exist for %s", node.ObjectMeta.Name)
}
ann := stage + "-" + encryptionConfigHash
node.Annotations[EncryptionHashAnnotation] = ann
if _, err = runtime.Core.Core().V1().Node().Update(node); err != nil {
return err
}
logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
return ioutil.WriteFile(runtime.EncryptionHash, []byte(ann), 0600)
}

196
pkg/secretsencrypt/controller.go Normal file
View File

@ -0,0 +1,196 @@
package secretsencrypt
import (
"context"
"fmt"
"strings"
"github.com/rancher/k3s/pkg/cluster"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/util"
coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/pager"
"k8s.io/client-go/tools/record"
)
const (
controllerAgentName string = "reencrypt-controller"
secretsUpdateStartEvent string = "SecretsUpdateStart"
secretsProgressEvent string = "SecretsProgress"
secretsUpdateCompleteEvent string = "SecretsUpdateComplete"
secretsUpdateErrorEvent string = "SecretsUpdateError"
)
type handler struct {
ctx context.Context
controlConfig *config.Control
nodes coreclient.NodeController
secrets coreclient.SecretController
recorder record.EventRecorder
}
func Register(
ctx context.Context,
k8s kubernetes.Interface,
controlConfig *config.Control,
nodes coreclient.NodeController,
secrets coreclient.SecretController,
) error {
h := &handler{
ctx: ctx,
controlConfig: controlConfig,
nodes: nodes,
secrets: secrets,
recorder: util.BuildControllerEventRecorder(k8s, controllerAgentName),
}
nodes.OnChange(ctx, "reencrypt-controller", h.onChangeNode)
return nil
}
// onChangeNode handles changes to Nodes. We are looking for a specific annotation change
func (h *handler) onChangeNode(key string, node *corev1.Node) (*corev1.Node, error) {
if node == nil {
return nil, nil
}
ann, ok := node.Annotations[EncryptionHashAnnotation]
if !ok {
return node, nil
}
if valid, err := h.validateReencryptStage(node, ann); err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
} else if !valid {
return node, nil
}
reencryptHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive)
if err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
}
ann = EncryptionReencryptActive + "-" + reencryptHash
node.Annotations[EncryptionHashAnnotation] = ann
node, err = h.nodes.Update(node)
if err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
}
if err := h.updateSecrets(node); err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
}
// If skipping, revert back to the previous stage
if h.controlConfig.EncryptSkip {
BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime)
if node, err := h.nodes.Update(node); err != nil {
return node, err
}
return node, nil
}
// Remove last key
curKeys, err := GetEncryptionKeys(h.controlConfig.Runtime)
if err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
}
curKeys = curKeys[:len(curKeys)-1]
if err = WriteEncryptionConfig(h.controlConfig.Runtime, curKeys, true); err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
}
logrus.Infoln("Removed key: ", curKeys[len(curKeys)-1])
if err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
}
if err := WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished); err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
}
if err := cluster.Save(h.ctx, h.controlConfig, h.controlConfig.Runtime.EtcdConfig, true); err != nil {
h.recorder.Event(node, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error())
return node, err
}
return node, nil
}
// validateReencryptStage ensures that the request for reencryption is valid and
// that there is only one active reencryption at a time
func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (bool, error) {
split := strings.Split(annotation, "-")
if len(split) != 2 {
err := fmt.Errorf("invalid annotation %s found on node %s", annotation, node.ObjectMeta.Name)
return false, err
}
stage := split[0]
hash := split[1]
// Validate the specific stage and the request via sha256 hash
if stage != EncryptionReencryptRequest {
return false, nil
}
if reencryptRequestHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptRequest); err != nil {
return false, err
} else if reencryptRequestHash != hash {
err = fmt.Errorf("invalid hash: %s found on node %s", hash, node.ObjectMeta.Name)
return false, err
}
nodes, err := h.nodes.List(metav1.ListOptions{})
if err != nil {
return false, err
}
reencryptActiveHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive)
if err != nil {
return false, err
}
for _, node := range nodes.Items {
if ann, ok := node.Annotations[EncryptionHashAnnotation]; ok {
split := strings.Split(ann, "-")
if len(split) != 2 {
return false, fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name)
}
stage := split[0]
hash := split[1]
if stage == EncryptionReencryptActive && hash == reencryptActiveHash {
return false, fmt.Errorf("another reencrypt is already active")
}
}
}
return true, nil
}
func (h *handler) updateSecrets(node *corev1.Node) error {
secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
return h.secrets.List("", opts)
}))
i := 0
secretPager.EachListItem(h.ctx, metav1.ListOptions{}, func(obj runtime.Object) error {
if secret, ok := obj.(*corev1.Secret); ok {
if _, err := h.secrets.Update(secret); err != nil {
return fmt.Errorf("failed to reencrypted secret: %v", err)
}
if i != 0 && i%10 == 0 {
h.recorder.Eventf(node, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i)
}
i++
}
return nil
})
h.recorder.Eventf(node, corev1.EventTypeNormal, secretsUpdateCompleteEvent, "completed reencrypt of %d secrets", i)
return nil
}

2
pkg/server/router.go
View File

@ -48,6 +48,8 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
authed.Path(prefix + "/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
authed.Path(prefix + "/config").Handler(configHandler(serverConfig, cfg))
authed.Path(prefix + "/readyz").Handler(readyzHandler(serverConfig))
authed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig))
authed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig))
nodeAuthed := mux.NewRouter()
nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes"))

379
pkg/server/secrets-encrypt.go Normal file
View File

@ -0,0 +1,379 @@
package server
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"math/big"
"net/http"
"os"
"strings"
"time"
"github.com/rancher/k3s/pkg/cluster"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/secretsencrypt"
"github.com/rancher/wrangler/pkg/generated/controllers/core"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
"k8s.io/utils/pointer"
)
const aescbcKeySize = 32
type EncryptionState struct {
Stage string `json:"stage"`
ActiveKey string `json:"activekey"`
Enable *bool `json:"enable,omitempty"`
HashMatch bool `json:"hashmatch,omitempty"`
HashError string `json:"hasherror,omitempty"`
InactiveKeys []string `json:"inactivekeys,omitempty"`
}
type EncryptionRequest struct {
Stage *string `json:"stage,omitempty"`
Enable *bool `json:"enable,omitempty"`
Force bool `json:"force"`
Skip bool `json:"skip"`
}
func getEncryptionRequest(req *http.Request) (EncryptionRequest, error) {
b, err := ioutil.ReadAll(req.Body)
if err != nil {
return EncryptionRequest{}, err
}
result := EncryptionRequest{}
err = json.Unmarshal(b, &result)
return result, err
}
func encryptionStatusHandler(server *config.Control) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
}
status, err := encryptionStatus(server)
if err != nil {
genErrorMessage(resp, http.StatusInternalServerError, err)
return
}
b, err := json.Marshal(status)
if err != nil {
genErrorMessage(resp, http.StatusInternalServerError, err)
return
}
resp.Write(b)
})
}
func encryptionStatus(server *config.Control) (EncryptionState, error) {
state := EncryptionState{}
providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime)
if os.IsNotExist(err) {
return state, nil
} else if err != nil {
return state, err
}
if providers[1].Identity != nil && providers[0].AESCBC != nil {
state.Enable = pointer.Bool(true)
} else if providers[0].Identity != nil && providers[1].AESCBC != nil || !server.EncryptSecrets {
state.Enable = pointer.Bool(false)
}
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), ""); err != nil {
state.HashMatch = false
state.HashError = err.Error()
} else {
state.HashMatch = true
}
stage, _, err := getEncryptionHashAnnotation(server.Runtime.Core.Core())
if err != nil {
return state, err
}
state.Stage = stage
active := true
for _, p := range providers {
if p.AESCBC != nil {
for _, aesKey := range p.AESCBC.Keys {
if active {
active = false
state.ActiveKey = aesKey.Name
} else {
state.InactiveKeys = append(state.InactiveKeys, aesKey.Name)
}
}
}
if p.Identity != nil {
active = false
}
}
return state, nil
}
func encryptionEnable(ctx context.Context, server *config.Control, enable bool) error {
providers, err := secretsencrypt.GetEncryptionProviders(server.Runtime)
if err != nil {
return err
}
if len(providers) > 2 {
return fmt.Errorf("more than 2 providers (%d) found in secrets encryption", len(providers))
}
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
if err != nil {
return err
}
if providers[1].Identity != nil && providers[0].AESCBC != nil && !enable {
logrus.Infoln("Disabling secrets encryption")
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
return err
}
} else if !enable {
logrus.Infoln("Secrets encryption already disabled")
return nil
} else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable {
logrus.Infoln("Enabling secrets encryption")
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
return err
}
} else if enable {
logrus.Infoln("Secrets encryption already enabled")
return nil
} else {
return fmt.Errorf("unable to enable/disable secrets encryption, unknown configuration")
}
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
}
func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
}
if req.Method != http.MethodPut {
resp.WriteHeader(http.StatusBadRequest)
return
}
encryptReq, err := getEncryptionRequest(req)
if err != nil {
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(err.Error()))
return
}
if encryptReq.Stage != nil {
switch *encryptReq.Stage {
case secretsencrypt.EncryptionPrepare:
err = encryptionPrepare(ctx, server, encryptReq.Force)
case secretsencrypt.EncryptionRotate:
err = encryptionRotate(ctx, server, encryptReq.Force)
case secretsencrypt.EncryptionReencryptActive:
err = encryptionReencrypt(ctx, server, encryptReq.Force, encryptReq.Skip)
default:
err = fmt.Errorf("unknown stage %s requested", *encryptReq.Stage)
}
} else if encryptReq.Enable != nil {
err = encryptionEnable(ctx, server, *encryptReq.Enable)
}
if err != nil {
genErrorMessage(resp, http.StatusBadRequest, err)
return
}
resp.WriteHeader(http.StatusOK)
})
}
func encryptionPrepare(ctx context.Context, server *config.Control, force bool) error {
states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil && !force {
return err
}
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
if err != nil {
return err
}
if err := AppendNewEncryptionKey(&curKeys); err != nil {
return err
}
logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1])
if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil {
return err
}
nodeName := os.Getenv("NODE_NAME")
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
if err != nil {
return err
}
if err = secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare); err != nil {
return err
}
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
}
func encryptionRotate(ctx context.Context, server *config.Control, force bool) error {
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionPrepare); err != nil && !force {
return err
}
curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
if err != nil {
return err
}
// Right rotate elements
rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...)
if err = secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true); err != nil {
return err
}
logrus.Infoln("Encryption keys right rotated")
nodeName := os.Getenv("NODE_NAME")
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
if err != nil {
return err
}
if err := secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate); err != nil {
return err
}
return cluster.Save(ctx, server, server.Runtime.EtcdConfig, true)
}
func encryptionReencrypt(ctx context.Context, server *config.Control, force bool, skip bool) error {
if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionRotate); err != nil && !force {
return err
}
server.EncryptForce = force
server.EncryptSkip = skip
nodeName := os.Getenv("NODE_NAME")
node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
if err != nil {
return err
}
reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest)
if err != nil {
return err
}
ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash
node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann
if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil {
return err
}
logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
return nil
}
func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error {
aescbcKey := make([]byte, aescbcKeySize)
_, err := rand.Read(aescbcKey)
if err != nil {
return err
}
encodedKey := base64.StdEncoding.EncodeToString(aescbcKey)
newKey := []apiserverconfigv1.Key{
{
Name: "aescbckey-" + time.Now().Format(time.RFC3339),
Secret: encodedKey,
},
}
*keys = append(*keys, newKey...)
return nil
}
func getEncryptionHashAnnotation(core core.Interface) (string, string, error) {
nodeName := os.Getenv("NODE_NAME")
node, err := core.V1().Node().Get(nodeName, metav1.GetOptions{})
if err != nil {
return "", "", err
}
ann := node.Annotations[secretsencrypt.EncryptionHashAnnotation]
split := strings.Split(ann, "-")
if len(split) != 2 {
return "", "", fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name)
}
return split[0], split[1], nil
}
func getServerNodes(core core.Interface) ([]corev1.Node, error) {
nodes, err := core.V1().Node().List(metav1.ListOptions{})
if err != nil {
return nil, err
}
var serverNodes []corev1.Node
for _, node := range nodes.Items {
if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" {
serverNodes = append(serverNodes, node)
}
}
return serverNodes, nil
}
// verifyEncryptionHashAnnotation checks that all nodes are on the same stage,
// and that a request for new stage is valid
func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.Interface, prevStage string) error {
var firstHash string
var firstNodeName string
first := true
serverNodes, err := getServerNodes(core)
if err != nil {
return err
}
for _, node := range serverNodes {
hash, ok := node.Annotations[secretsencrypt.EncryptionHashAnnotation]
if ok && first {
firstHash = hash
first = false
firstNodeName = node.ObjectMeta.Name
} else if ok && hash != firstHash {
return fmt.Errorf("hash does not match between %s and %s", firstNodeName, node.ObjectMeta.Name)
}
}
if prevStage == "" {
return nil
}
oldStage, oldHash, err := getEncryptionHashAnnotation(core)
if err != nil {
return err
}
encryptionConfigHash, err := secretsencrypt.GenEncryptionConfigHash(runtime)
if err != nil {
return err
}
if !strings.Contains(prevStage, oldStage) {
return fmt.Errorf("incorrect stage: %s found on node %s", oldStage, serverNodes[0].ObjectMeta.Name)
} else if oldHash != encryptionConfigHash {
return fmt.Errorf("invalid hash: %s found on node %s", oldHash, serverNodes[0].ObjectMeta.Name)
}
return nil
}
func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error) {
errID, err := rand.Int(rand.Reader, big.NewInt(99999))
if err != nil {
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(err.Error()))
return
}
logrus.Warnf("secrets-encrypt-%s: %s", errID.String(), passedErr.Error())
resp.WriteHeader(statusCode)
resp.Write([]byte(fmt.Sprintf("error secrets-encrypt-%s: see server logs for more info", errID.String())))
}

34
pkg/server/server.go
View File

@ -25,6 +25,7 @@ import (
"github.com/rancher/k3s/pkg/node"
"github.com/rancher/k3s/pkg/nodepassword"
"github.com/rancher/k3s/pkg/rootlessports"
"github.com/rancher/k3s/pkg/secretsencrypt"
"github.com/rancher/k3s/pkg/servicelb"
"github.com/rancher/k3s/pkg/static"
"github.com/rancher/k3s/pkg/util"
@ -169,7 +170,7 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err
}
}
go setControlPlaneRoleLabel(ctx, sc.Core.Core().V1().Node(), config)
go setNodeLabelsAndAnnotations(ctx, sc.Core.Core().V1().Node(), config)
go setClusterDNSConfig(ctx, config, sc.Core.Core().V1().ConfigMap())
@ -232,6 +233,16 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
return err
}
if config.ControlConfig.EncryptSecrets {
if err := secretsencrypt.Register(ctx,
sc.K8s,
&config.ControlConfig,
sc.Core.Core().V1().Node(),
sc.Core.Core().V1().Secret()); err != nil {
return err
}
}
if config.Rootless {
return rootlessports.Register(ctx,
sc.Core.Core().V1().Service(),
@ -490,7 +501,7 @@ func isSymlink(config string) bool {
return false
}
func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient, config *Config) error {
func setNodeLabelsAndAnnotations(ctx context.Context, nodes v1.NodeClient, config *Config) error {
if config.DisableAgent || config.ControlConfig.DisableAPIServer {
return nil
}
@ -515,18 +526,25 @@ func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient, config *
etcdRoleLabelExists = true
}
}
if v, ok := node.Labels[ControlPlaneRoleLabelKey]; ok && v == "true" && !etcdRoleLabelExists {
break
}
if node.Labels == nil {
node.Labels = make(map[string]string)
}
node.Labels[ControlPlaneRoleLabelKey] = "true"
node.Labels[MasterRoleLabelKey] = "true"
v, ok := node.Labels[ControlPlaneRoleLabelKey]
if !ok || v != "true" || etcdRoleLabelExists {
node.Labels[ControlPlaneRoleLabelKey] = "true"
node.Labels[MasterRoleLabelKey] = "true"
}
if config.ControlConfig.EncryptSecrets {
if err = secretsencrypt.BootstrapEncryptionHashAnnotation(node, config.ControlConfig.Runtime); err != nil {
logrus.Infof("Unable to set encryption hash annotation %s", err.Error())
break
}
}
_, err = nodes.Update(node)
if err == nil {
logrus.Infof("Control-plane role label has been set successfully on node: %s", nodeName)
logrus.Infof("Labels and annotations have been set successfully on node: %s", nodeName)
break
}
select {

14
pkg/util/api.go
View File

@ -5,15 +5,20 @@ import (
"fmt"
"net"
"net/http"
"os"
"strconv"
"time"
"github.com/pkg/errors"
"github.com/rancher/wrangler/pkg/merr"
"github.com/rancher/wrangler/pkg/schemes"
"github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
coregetter "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record"
)
// This sets a default duration to wait for the apiserver to become ready. This is primarily used to
@ -72,3 +77,12 @@ func WaitForAPIServerReady(ctx context.Context, client clientset.Interface, time
return nil
}
func BuildControllerEventRecorder(k8s clientset.Interface, controllerName string) record.EventRecorder {
logrus.Infof("Creating %s event broadcaster", controllerName)
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(logrus.Infof)
eventBroadcaster.StartRecordingToSink(&coregetter.EventSinkImpl{Interface: k8s.CoreV1().Events(metav1.NamespaceSystem)})
nodeName := os.Getenv("NODE_NAME")
return eventBroadcaster.NewRecorder(schemes.All, v1.EventSource{Component: controllerName, Host: nodeName})
}

2
scripts/build
View File

@ -78,6 +78,7 @@ rm -f \
bin/containerd-shim-runc-v2 \
bin/k3s-server \
bin/k3s-etcd-snapshot \
bin/k3s-secrets-encrypt \
bin/k3s-certificate \
bin/kubectl \
bin/crictl \
@ -108,6 +109,7 @@ CGO_ENABLED=1 "${GO}" build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STAT
ln -s containerd ./bin/k3s-agent
ln -s containerd ./bin/k3s-server
ln -s containerd ./bin/k3s-etcd-snapshot
ln -s containerd ./bin/k3s-secrets-encrypt
ln -s containerd ./bin/k3s-certificate
ln -s containerd ./bin/kubectl
ln -s containerd ./bin/crictl

2
scripts/package-cli
View File

@ -7,7 +7,7 @@ cd $(dirname $0)/..
GO=${GO-go}
for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-certificate k3s ; do
for i in crictl kubectl k3s-agent k3s-server k3s-etcd-snapshot k3s-secrets-encrypt k3s-certificate k3s ; do
rm -f bin/$i
ln -s containerd bin/$i
done

7
tests/integration/localstorage/localstorage_int_test.go
View File

@ -15,7 +15,6 @@ import (
var localStorageServer *testutil.K3sServer
var localStorageServerArgs = []string{"--cluster-init"}
var testDataDir = "../../testdata/"
var _ = BeforeSuite(func() {
if !testutil.IsExistingServer() {
var err error
@ -37,12 +36,12 @@ var _ = Describe("local storage", func() {
}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
})
It("creates a new pvc", func() {
result, err := testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pvc.yaml")
result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pvc.yaml")
Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created"))
Expect(err).NotTo(HaveOccurred())
})
It("creates a new pod", func() {
Expect(testutil.K3sCmd("kubectl", "create", "-f", testDataDir+"localstorage_pod.yaml")).
Expect(testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pod.yaml")).
To(ContainSubstring("pod/volume-test created"))
})
It("shows storage up in kubectl", func() {
@ -51,7 +50,7 @@ var _ = Describe("local storage", func() {
}, "45s", "1s").Should(MatchRegexp(`local-path-pvc.+Bound`))
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv")
}, "10s", "1s").Should(MatchRegexp(`pvc.+2Gi.+Bound`))
}, "10s", "1s").Should(MatchRegexp(`pvc.+1Gi.+Bound`))
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pod")
}, "10s", "1s").Should(MatchRegexp(`volume-test.+Running`))

0
tests/testdata/localstorage_pod.yaml → tests/integration/localstorage/testdata/localstorage_pod.yaml vendored
View File

2
tests/testdata/localstorage_pvc.yaml → tests/integration/localstorage/testdata/localstorage_pvc.yaml vendored
View File

@ -9,4 +9,4 @@ spec:
storageClassName: local-path
resources:
requests:
storage: 2Gi
storage: 1Gi

155
tests/integration/secretsencryption/secretsencryption_int_test.go Normal file
View File

@ -0,0 +1,155 @@
package integration
import (
"fmt"
"os"
"regexp"
"testing"
"time"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/gomega"
testutil "github.com/rancher/k3s/tests/util"
)
var secretsEncryptionServer *testutil.K3sServer
var secretsEncryptionDataDir = "/tmp/k3sse"
var secretsEncryptionServerArgs = []string{"--secrets-encryption", "-d", secretsEncryptionDataDir}
var _ = BeforeSuite(func() {
if !testutil.IsExistingServer() {
var err error
Expect(os.MkdirAll(secretsEncryptionDataDir, 0777)).To(Succeed())
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred())
}
})
var _ = Describe("secrets encryption rotation", func() {
BeforeEach(func() {
if testutil.IsExistingServer() {
Skip("Test does not support running on existing k3s servers")
}
})
When("A server starts with secrets encryption", func() {
It("starts up with no problems", func() {
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
})
It("it creates a encryption key", func() {
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Encryption Status: Enabled"))
Expect(result).To(ContainSubstring("Current Rotation Stage: start"))
})
})
When("A server rotates encryption keys", func() {
It("it prepares to rotate", func() {
Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)).
To(ContainSubstring("prepare completed successfully"))
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Current Rotation Stage: prepare"))
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
Expect(err).ToNot(HaveOccurred())
keys := reg.FindAllString(result, -1)
Expect(keys).To(HaveLen(2))
Expect(keys[0]).To(ContainSubstring("aescbckey"))
Expect(keys[1]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
})
It("restarts the server", func() {
var err error
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred())
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
})
It("rotates the keys", func() {
Eventually(func() (string, error) {
return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir)
}, "10s", "2s").Should(ContainSubstring("rotate completed successfully"))
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Current Rotation Stage: rotate"))
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
Expect(err).ToNot(HaveOccurred())
keys := reg.FindAllString(result, -1)
Expect(keys).To(HaveLen(2))
Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
Expect(keys[1]).To(ContainSubstring("aescbckey"))
})
It("restarts the server", func() {
var err error
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred())
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
time.Sleep(10 * time.Second)
})
It("reencrypts the keys", func() {
Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)).
To(ContainSubstring("reencryption started"))
Eventually(func() (string, error) {
return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
Expect(err).ToNot(HaveOccurred())
keys := reg.FindAllString(result, -1)
Expect(keys).To(HaveLen(1))
Expect(keys[0]).To(ContainSubstring("aescbckey-" + fmt.Sprint(time.Now().Year())))
})
})
When("A server disables encryption", func() {
It("it triggers the disable", func() {
Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)).
To(ContainSubstring("secrets-encryption disabled"))
result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
})
It("restarts the server", func() {
var err error
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred())
Eventually(func() (string, error) {
return testutil.K3sCmd("kubectl", "get", "pods", "-A")
}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
time.Sleep(10 * time.Second)
})
It("reencrypts the keys", func() {
result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("reencryption started"))
result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
})
})
})
var _ = AfterSuite(func() {
if !testutil.IsExistingServer() {
Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
Expect(testutil.K3sRemoveDataDir(secretsEncryptionDataDir)).To(Succeed())
}
})
func Test_IntegrationSecretsEncryption(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Secrets Encryption Suite", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-se.xml"),
})
}

20
tests/util/cmd.go
View File

@ -80,6 +80,18 @@ func K3sCmd(cmdName string, cmdArgs ...string) (string, error) {
return string(byteOut), err
}
// K3sRemoveDataDir removes the provided directory as root
func K3sRemoveDataDir(dataDir string) error {
var cmd *exec.Cmd
if IsRoot() {
cmd = exec.Command("rm", "-rf", dataDir)
} else {
cmd = exec.Command("sudo", "rm", "-rf", dataDir)
}
_, err := cmd.CombinedOutput()
return err
}
func contains(source []string, target string) bool {
for _, s := range source {
if s == target {
@ -168,11 +180,5 @@ func K3sKillServer(server *K3sServer) error {
return err
}
}
if err := flock.Release(server.lock); err != nil {
return err
}
if !flock.CheckLock(lockFile) {
return os.RemoveAll(lockFile)
}
return nil
return flock.Release(server.lock)
}