Mirror
/
k3s
mirror of https://github.com/k3s-io/k3s.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Server Token Rotation (#8265) 

* Consolidate NewCertCommands
* Add support for user defined new token
* Add E2E testlets

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Ensure agent token also changes

Signed-off-by: Derek Nola <derek.nola@suse.com>
This commit is contained in:
Derek Nola 2023-10-09 10:58:49 -07:00 committed by GitHub
parent ced25af5b1
commit dface01de8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 325 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/cert/main.go
View File

@ -15,11 +15,9 @@ import (
func main() {
app := cmds.NewApp()
app.Commands = []cli.Command{
cmds.NewCertCommand(
cmds.NewCertSubcommands(
cert.Rotate,
cert.RotateCA,
),
cmds.NewCertCommands(
cert.Rotate,
cert.RotateCA,
),
}

9
cmd/k3s/main.go
View File

@ -57,6 +57,7 @@ func main() {
tokenCommand,
tokenCommand,
tokenCommand,
tokenCommand,
),
cmds.NewEtcdSnapshotCommands(
etcdsnapshotCommand,
@ -73,11 +74,9 @@ func main() {
secretsencryptCommand,
secretsencryptCommand,
),
cmds.NewCertCommand(
cmds.NewCertSubcommands(
certCommand,
certCommand,
),
cmds.NewCertCommands(
certCommand,
certCommand,
),
cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)),
}

9
cmd/server/main.go
View File

@ -54,6 +54,7 @@ func main() {
token.Delete,
token.Generate,
token.List,
token.Rotate,
),
cmds.NewEtcdSnapshotCommands(
etcdsnapshot.Delete,
@ -70,11 +71,9 @@ func main() {
secretsencrypt.Reencrypt,
secretsencrypt.RotateKeys,
),
cmds.NewCertCommand(
cmds.NewCertSubcommands(
cert.Rotate,
cert.RotateCA,
),
cmds.NewCertCommands(
cert.Rotate,
cert.RotateCA,
),
cmds.NewCompletionCommand(completion.Run),
}

1
cmd/token/main.go
View File

@ -20,6 +20,7 @@ func main() {
token.Delete,
token.Generate,
token.List,
token.Rotate,
),
}

8
main.go
View File

@ -47,11 +47,9 @@ func main() {
secretsencrypt.Reencrypt,
secretsencrypt.RotateKeys,
),
cmds.NewCertCommand(
cmds.NewCertSubcommands(
cert.Rotate,
cert.RotateCA,
),
cmds.NewCertCommands(
cert.Rotate,
cert.RotateCA,
),
cmds.NewCompletionCommand(completion.Run),
}

40
pkg/cli/cmds/certs.go
View File

@ -54,33 +54,29 @@ var (
}
)
func NewCertCommand(subcommands []cli.Command) cli.Command {
func NewCertCommands(rotate, rotateCA func(ctx *cli.Context) error) cli.Command {
return cli.Command{
Name: CertCommand,
Usage: "Manage K3s certificates",
SkipFlagParsing: false,
SkipArgReorder: true,
Subcommands: subcommands,
}
}
func NewCertSubcommands(rotate, rotateCA func(ctx *cli.Context) error) []cli.Command {
return []cli.Command{
{
Name: "rotate",
Usage: "Rotate " + version.Program + " component certificates on disk",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: rotate,
Flags: CertRotateCommandFlags,
},
{
Name: "rotate-ca",
Usage: "Write updated " + version.Program + " CA certificates to the datastore",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: rotateCA,
Flags: CertRotateCACommandFlags,
Subcommands: []cli.Command{
{
Name: "rotate",
Usage: "Rotate " + version.Program + " component certificates on disk",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: rotate,
Flags: CertRotateCommandFlags,
},
{
Name: "rotate-ca",
Usage: "Write updated " + version.Program + " CA certificates to the datastore",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: rotateCA,
Flags: CertRotateCACommandFlags,
},
},
}
}

31
pkg/cli/cmds/token.go
View File

@ -3,6 +3,7 @@ package cmds
import (
"time"
"github.com/k3s-io/k3s/pkg/version"
"github.com/urfave/cli"
)
@ -12,7 +13,9 @@ const TokenCommand = "token"
type Token struct {
Description string
Kubeconfig string
ServerURL string
Token string
NewToken string
Output string
Groups cli.StringSlice
Usages cli.StringSlice
@ -32,7 +35,7 @@ var (
}
)
func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) error) cli.Command {
func NewTokenCommands(create, delete, generate, list, rotate func(ctx *cli.Context) error) cli.Command {
return cli.Command{
Name: TokenCommand,
Usage: "Manage bootstrap tokens",
@ -92,6 +95,32 @@ func NewTokenCommands(create, delete, generate, list func(ctx *cli.Context) erro
SkipArgReorder: true,
Action: list,
},
{
Name: "rotate",
Usage: "Rotate original server token with a new bootstrap token",
Flags: append(TokenFlags,
&cli.StringFlag{
Name: "token,t",
Usage: "Existing token used to join a server or agent to a cluster",
Destination: &TokenConfig.Token,
EnvVar: version.ProgramUpper + "_TOKEN",
},
&cli.StringFlag{
Name: "server, s",
Usage: "(cluster) Server to connect to",
Destination: &TokenConfig.ServerURL,
EnvVar: version.ProgramUpper + "_URL",
Value: "https://127.0.0.1:6443",
},
&cli.StringFlag{
Name: "new-token",
Usage: "New token that replaces existing token",
Destination: &TokenConfig.NewToken,
}),
SkipFlagParsing: false,
SkipArgReorder: true,
Action: rotate,
},
},
}
}

50
pkg/cli/token/token.go
View File

@ -1,18 +1,23 @@
package token
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"text/tabwriter"
"time"
"github.com/erikdubbelboer/gspt"
"github.com/k3s-io/k3s/pkg/cli/cmds"
"github.com/k3s-io/k3s/pkg/clientaccess"
"github.com/k3s-io/k3s/pkg/kubeadm"
"github.com/k3s-io/k3s/pkg/server"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/version"
"github.com/pkg/errors"
"github.com/urfave/cli"
"gopkg.in/yaml.v2"
@ -22,6 +27,7 @@ import (
"k8s.io/client-go/tools/clientcmd"
bootstrapapi "k8s.io/cluster-bootstrap/token/api"
bootstraputil "k8s.io/cluster-bootstrap/token/util"
"k8s.io/utils/pointer"
)
func Create(app *cli.Context) error {
@ -139,6 +145,50 @@ func generate(app *cli.Context, cfg *cmds.Token) error {
return nil
}
func Rotate(app *cli.Context) error {
if err := cmds.InitLogging(); err != nil {
return err
}
fmt.Println("\033[33mWARNING\033[0m: Recommended to keep a record of the old token. If restoring from a snapshot, you must use the token associated with that snapshot.")
info, err := serverAccess(&cmds.TokenConfig)
if err != nil {
return err
}
b, err := json.Marshal(server.TokenRotateRequest{
NewToken: pointer.String(cmds.TokenConfig.NewToken),
})
if err != nil {
return err
}
if err = info.Put("/v1-"+version.Program+"/token", b); err != nil {
return err
}
// wait for etcd db propagation delay
time.Sleep(1 * time.Second)
fmt.Println("Token rotated, restart k3s with new token")
return nil
}
func serverAccess(cfg *cmds.Token) (*clientaccess.Info, error) {
// hide process arguments from ps output, since they likely contain tokens.
gspt.SetProcTitle(os.Args[0] + " token")
dataDir, err := server.ResolveDataDir("")
if err != nil {
return nil, err
}
if cfg.Token == "" {
fp := filepath.Join(dataDir, "token")
tokenByte, err := os.ReadFile(fp)
if err != nil {
return nil, err
}
cfg.Token = string(bytes.TrimRight(tokenByte, "\n"))
}
return clientaccess.ParseAndValidateToken(cfg.ServerURL, cfg.Token, clientaccess.WithUser("server"))
}
func List(app *cli.Context) error {
if err := cmds.InitLogging(); err != nil {
return err

55
pkg/cluster/storage.go
View File

@ -21,6 +21,49 @@ import (
// After this many attempts, the lock is deleted and the counter reset.
const maxBootstrapWaitAttempts = 5
func RotateBootstrapToken(ctx context.Context, config *config.Control, oldToken string) error {
token, err := readTokenFromFile(config.Runtime.ServerToken, config.Runtime.ServerCA, config.DataDir)
if err != nil {
return err
}
normalizedToken, err := normalizeToken(token)
if err != nil {
return err
}
storageClient, err := client.New(config.Runtime.EtcdConfig)
if err != nil {
return err
}
defer storageClient.Close()
tokenKey := storageKey(normalizedToken)
var bootstrapList []client.Value
if err := wait.PollImmediateUntilWithContext(ctx, 5*time.Second, func(ctx context.Context) (bool, error) {
bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0)
if err != nil {
return false, err
}
return true, nil
}); err != nil {
return err
}
normalizedOldToken, err := normalizeToken(oldToken)
if err != nil {
return err
}
// reuse the existing migration function to reencrypt bootstrap data with new token
if err := migrateTokens(ctx, bootstrapList, storageClient, "", tokenKey, normalizedToken, normalizedOldToken); err != nil {
return err
}
return nil
}
// Save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete
// snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token.
// This is used when bootstrapping a cluster from a managed database or external etcd cluster.
@ -225,7 +268,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
logrus.Warn("found multiple bootstrap keys in storage")
}
// check for empty string key and for old token format with k10 prefix
if err := migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
if err := migrateTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil {
return nil, false, err
}
@ -236,6 +279,7 @@ func getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client
}
for _, bootstrapKV := range bootstrapList {
// ensure bootstrap is stored in the current token's key
logrus.Debugf("checking bootstrap key %s against %s", string(bootstrapKV.Key), tokenKey)
if string(bootstrapKV.Key) == tokenKey {
return &bootstrapKV, false, nil
}
@ -277,21 +321,24 @@ func normalizeToken(token string) (string, error) {
return password, nil
}
// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is
// migrateTokens will list all keys that has prefix /bootstrap and will check for key that is
// hashed with empty string and keys that is hashed with old token format before normalizing
// then migrate those and resave only with the normalized token
func migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
func migrateTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error {
oldTokenKey := storageKey(oldToken)
for _, bootstrapKV := range bootstrapList {
// checking for empty string bootstrap key
logrus.Debug("Comparing ", string(bootstrapKV.Key), " to ", oldTokenKey)
if string(bootstrapKV.Key) == emptyStringKey {
logrus.Warn("Bootstrap data encrypted with empty string, deleting and resaving with token")
if err := doMigrateToken(ctx, storageClient, bootstrapKV, "", emptyStringKey, token, tokenKey); err != nil {
return err
}
} else if string(bootstrapKV.Key) == oldTokenKey && oldTokenKey != tokenKey {
logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
if emptyStringKey != "" {
logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token")
}
if err := doMigrateToken(ctx, storageClient, bootstrapKV, oldToken, oldTokenKey, token, tokenKey); err != nil {
return err
}

1
pkg/daemons/control/deps/deps.go
View File

@ -243,6 +243,7 @@ func genUsers(config *config.Control) error {
return err
}
// if no token is provided on bootstrap, we generate a random token
serverPass, err := getServerPass(passwd, config)
if err != nil {
return err

1
pkg/server/router.go
View File

@ -85,6 +85,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
serverAuthed.Path("/db/info").Handler(nodeAuthed)
serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime))
serverAuthed.Path(prefix + "/token").Handler(tokenRequestHandler(ctx, serverConfig))
systemAuthed := mux.NewRouter().SkipClean(true)
systemAuthed.NotFoundHandler = serverAuthed

100
pkg/server/token.go Normal file
View File

@ -0,0 +1,100 @@
package server
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"path/filepath"
"github.com/k3s-io/k3s/pkg/cluster"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/passwd"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/version"
"github.com/sirupsen/logrus"
)
type TokenRotateRequest struct {
NewToken *string `json:"newToken,omitempty"`
}
func getServerTokenRequest(req *http.Request) (TokenRotateRequest, error) {
b, err := io.ReadAll(req.Body)
if err != nil {
return TokenRotateRequest{}, err
}
result := TokenRotateRequest{}
err = json.Unmarshal(b, &result)
return result, err
}
func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil || req.Method != http.MethodPut {
resp.WriteHeader(http.StatusBadRequest)
return
}
var err error
sTokenReq, err := getServerTokenRequest(req)
logrus.Debug("Received token request")
if err != nil {
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(err.Error()))
return
}
if err = tokenRotate(ctx, server, *sTokenReq.NewToken); err != nil {
genErrorMessage(resp, http.StatusInternalServerError, err, "token")
return
}
resp.WriteHeader(http.StatusOK)
})
}
func tokenRotate(ctx context.Context, server *config.Control, newToken string) error {
passwd, err := passwd.Read(server.Runtime.PasswdFile)
if err != nil {
return err
}
if err != nil {
return err
}
oldToken, found := passwd.Pass("server")
if !found {
return fmt.Errorf("server token not found")
}
if newToken == "" {
newToken, err = util.Random(16)
if err != nil {
return err
}
}
if err := passwd.EnsureUser("server", version.Program+":server", newToken); err != nil {
return err
}
// If the agent token is the same a server, we need to change both
if agentToken, found := passwd.Pass("node"); found && agentToken == oldToken && server.AgentToken == "" {
if err := passwd.EnsureUser("node", version.Program+":agent", newToken); err != nil {
return err
}
}
if err := passwd.Write(server.Runtime.PasswdFile); err != nil {
return err
}
serverTokenFile := filepath.Join(server.DataDir, "token")
if err := writeToken("server:"+newToken, serverTokenFile, server.Runtime.ServerCA); err != nil {
return err
}
if err := cluster.RotateBootstrapToken(ctx, server, oldToken); err != nil {
return err
}
server.Token = newToken
return cluster.Save(ctx, server, true)
}

14
tests/e2e/token/Vagrantfile vendored
View File

@ -1,6 +1,6 @@
ENV['VAGRANT_NO_PARALLEL'] = 'no'
NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
["server-0", "agent-0", "agent-1" ])
["server-0", "server-1", "server-2", "agent-0", "agent-1"])
NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
@ -40,6 +40,18 @@ def provision(vm, roles, role_num, node_num)
YAML
k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
end
elsif roles.include?("server") && role_num != 0
vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s|
k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
k3s.args = "server"
k3s.config = <<~YAML
server: "https://#{NETWORK_PREFIX}.100:6443"
token: vagrant
node-external-ip: #{node_ip}
flannel-iface: eth1
YAML
k3s.env = ["K3S_KUBECONFIG_MODE=0644", install_type]
end
end
if roles.include?("agent")
vm.provision :k3s, run: 'once' do |k3s|

50
tests/e2e/token/token_test.go
View File

@ -17,7 +17,7 @@ import (
// generic/ubuntu2004, generic/centos7, generic/rocky8, opensuse/Leap-15.4.x86_64
var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
var serverCount = flag.Int("serverCount", 1, "number of server nodes")
var serverCount = flag.Int("serverCount", 3, "number of server nodes")
var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
var ci = flag.Bool("ci", false, "running on CI")
var local = flag.Bool("local", false, "deploy a locally built K3s binary")
@ -104,7 +104,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(2))
g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 1))
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
@ -122,7 +122,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
It("Cleans up 20s token automatically", func() {
Eventually(func() (string, error) {
return e2e.RunCmdOnNode("k3s token list", serverNodeNames[0])
}, "20s", "5s").ShouldNot(ContainSubstring("20sect"))
}, "25s", "5s").ShouldNot(ContainSubstring("20sect"))
})
var tempToken string
It("Creates a 10m agent token", func() {
@ -144,7 +144,49 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(3))
g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2))
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "60s", "5s").Should(Succeed())
})
})
Context("Rotate server bootstrap token", func() {
serverToken := "1234"
It("Creates a new server token", func() {
Expect(e2e.RunCmdOnNode("k3s token rotate -t vagrant --new-token="+serverToken, serverNodeNames[0])).
To(ContainSubstring("Token rotated, restart k3s with new token"))
})
It("Restarts servers with the new token", func() {
cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken)
for _, node := range serverNodeNames {
_, err := e2e.RunCmdOnNode(cmd, node)
Expect(err).NotTo(HaveOccurred())
}
for _, node := range serverNodeNames {
_, err := e2e.RunCmdOnNode("systemctl restart k3s", node)
Expect(err).NotTo(HaveOccurred())
}
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2))
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "60s", "5s").Should(Succeed())
})
It("Rejoins an agent with the new server token", func() {
cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken)
_, err := e2e.RunCmdOnNode(cmd, agentNodeNames[0])
Expect(err).NotTo(HaveOccurred())
_, err = e2e.RunCmdOnNode("systemctl restart k3s-agent", agentNodeNames[0])
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2))
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}