Add support for kubeadm token and client certificate auth

Allow bootstrapping with kubeadm bootstrap token strings or existing
Kubelet certs. This allows agents to join the cluster using kubeadm
bootstrap tokens, as created with the `k3s token create` command.

When the token expires or is deleted, agents can successfully restart by
authenticating with their kubelet certificate via node authentication.
If the token is gone and the node is deleted from the cluster, node auth
will fail and they will be prevented from rejoining the cluster until
provided with a valid token.

Servers still must be bootstrapped with the static cluster token, as
they will need to know it to decrypt the bootstrap data.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
This commit is contained in:
Brad Davidson 2022-12-08 23:59:21 +00:00 committed by Brad Davidson
parent 373df1c8b0
commit 992e64993d
9 changed files with 173 additions and 71 deletions

View File

@ -101,7 +101,7 @@ RETRY:
}
}
type HTTPRequester func(u string, client *http.Client, username, password string) ([]byte, error)
type HTTPRequester func(u string, client *http.Client, username, password, token string) ([]byte, error)
func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]byte, error) {
u, err := url.Parse(info.BaseURL)
@ -109,17 +109,19 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b
return nil, err
}
u.Path = path
return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts), info.Username, info.Password)
return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token())
}
func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string) HTTPRequester {
return func(u string, client *http.Client, username, password string) ([]byte, error) {
return func(u string, client *http.Client, username, password, token string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, u, nil)
if err != nil {
return nil, err
}
if username != "" {
if token != "" {
req.Header.Add("Authorization", "Bearer "+token)
} else if username != "" {
req.SetBasicAuth(username, password)
}
@ -320,8 +322,10 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
if envInfo.Debug {
logrus.SetLevel(logrus.DebugLevel)
}
info, err := clientaccess.ParseAndValidateToken(proxy.SupervisorURL(), envInfo.Token)
clientKubeletCert := filepath.Join(envInfo.DataDir, "agent", "client-kubelet.crt")
clientKubeletKey := filepath.Join(envInfo.DataDir, "agent", "client-kubelet.key")
withCert := clientaccess.WithClientCertificate(clientKubeletCert, clientKubeletKey)
info, err := clientaccess.ParseAndValidateToken(proxy.SupervisorURL(), envInfo.Token, withCert)
if err != nil {
return nil, err
}
@ -399,8 +403,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
return nil, err
}
clientKubeletCert := filepath.Join(envInfo.DataDir, "agent", "client-kubelet.crt")
clientKubeletKey := filepath.Join(envInfo.DataDir, "agent", "client-kubelet.key")
if err := getNodeNamedHostFile(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
return nil, err
}
@ -447,6 +449,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
nodeConfig.Images = filepath.Join(envInfo.DataDir, "agent", "images")
nodeConfig.AgentConfig.NodeName = nodeName
nodeConfig.AgentConfig.NodeConfigPath = nodeConfigPath
nodeConfig.AgentConfig.ClientKubeletCert = clientKubeletCert
nodeConfig.AgentConfig.ClientKubeletKey = clientKubeletKey
nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey
nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
@ -603,7 +607,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
// getAPIServers attempts to return a list of apiservers from the server.
func getAPIServers(ctx context.Context, node *config.Node, proxy proxy.Proxy) ([]string, error) {
info, err := clientaccess.ParseAndValidateToken(proxy.SupervisorURL(), node.Token)
withCert := clientaccess.WithClientCertificate(node.AgentConfig.ClientKubeletCert, node.AgentConfig.ClientKubeletKey)
info, err := clientaccess.ParseAndValidateToken(proxy.SupervisorURL(), node.Token, withCert)
if err != nil {
return nil, err
}
@ -620,7 +625,8 @@ func getAPIServers(ctx context.Context, node *config.Node, proxy proxy.Proxy) ([
// getKubeProxyDisabled attempts to return the DisableKubeProxy setting from the server configuration data.
// It first checks the server readyz endpoint, to ensure that the configuration has stabilized before use.
func getKubeProxyDisabled(ctx context.Context, node *config.Node, proxy proxy.Proxy) (bool, error) {
info, err := clientaccess.ParseAndValidateToken(proxy.SupervisorURL(), node.Token)
withCert := clientaccess.WithClientCertificate(node.AgentConfig.ClientKubeletCert, node.AgentConfig.ClientKubeletKey)
info, err := clientaccess.ParseAndValidateToken(proxy.SupervisorURL(), node.Token, withCert)
if err != nil {
return false, err
}

View File

@ -266,6 +266,9 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
func createProxyAndValidateToken(ctx context.Context, cfg *cmds.Agent) (proxy.Proxy, error) {
agentDir := filepath.Join(cfg.DataDir, "agent")
clientKubeletCert := filepath.Join(agentDir, "client-kubelet.crt")
clientKubeletKey := filepath.Join(agentDir, "client-kubelet.key")
if err := os.MkdirAll(agentDir, 0700); err != nil {
return nil, err
}
@ -276,8 +279,13 @@ func createProxyAndValidateToken(ctx context.Context, cfg *cmds.Agent) (proxy.Pr
return nil, err
}
options := []clientaccess.ValidationOption{
clientaccess.WithUser("node"),
clientaccess.WithClientCertificate(clientKubeletCert, clientKubeletKey),
}
for {
newToken, err := clientaccess.ParseAndValidateTokenForUser(proxy.SupervisorURL(), cfg.Token, "node")
newToken, err := clientaccess.ParseAndValidateToken(proxy.SupervisorURL(), cfg.Token, options...)
if err != nil {
logrus.Error(err)
select {

View File

@ -1,8 +1,10 @@
package agent
import (
"crypto/tls"
"fmt"
"os"
"path/filepath"
"runtime"
"github.com/erikdubbelboer/gspt"
@ -45,7 +47,11 @@ func Run(ctx *cli.Context) error {
cmds.AgentConfig.Token = token
}
if cmds.AgentConfig.Token == "" {
clientKubeletCert := filepath.Join(cmds.AgentConfig.DataDir, "agent", "client-kubelet.crt")
clientKubeletKey := filepath.Join(cmds.AgentConfig.DataDir, "agent", "client-kubelet.key")
_, err := tls.LoadX509KeyPair(clientKubeletCert, clientKubeletKey)
if err != nil && cmds.AgentConfig.Token == "" {
return fmt.Errorf("--token is required")
}

View File

@ -285,7 +285,7 @@ func rotateCA(app *cli.Context, cfg *cmds.Server, sync *cmds.CertRotateCA) error
return err
}
info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, serverConfig.ControlConfig.Token, "server")
info, err := clientaccess.ParseAndValidateToken(cmds.ServerConfig.ServerURL, serverConfig.ControlConfig.Token, clientaccess.WithUser("server"))
if err != nil {
return err
}

View File

@ -38,7 +38,7 @@ func commandPrep(app *cli.Context, cfg *cmds.Server) (*clientaccess.Info, error)
}
cfg.Token = string(bytes.TrimRight(tokenByte, "\n"))
}
return clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, cfg.Token, "server")
return clientaccess.ParseAndValidateToken(cmds.ServerConfig.ServerURL, cfg.Token, clientaccess.WithUser("server"))
}
func wrapServerError(err error) error {

View File

@ -14,6 +14,7 @@ import (
"strings"
"time"
"github.com/k3s-io/k3s/pkg/kubeadm"
"github.com/pkg/errors"
certutil "github.com/rancher/dynamiclistener/cert"
"github.com/sirupsen/logrus"
@ -21,7 +22,6 @@ import (
const (
tokenPrefix = "K10"
tokenFormat = "%s%s::%s:%s"
caHashLength = sha256.Size * 2
defaultClientTimeout = 10 * time.Second
@ -43,44 +43,66 @@ var (
// Info contains fields that track parsed parts of a cluster join token
type Info struct {
CACerts []byte `json:"cacerts,omitempty"`
BaseURL string `json:"baseurl,omitempty"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
*kubeadm.BootstrapTokenString
CACerts []byte
BaseURL string
Username string
Password string
CertFile string
KeyFile string
caHash string
}
// String returns the token data, templated according to the token format
// ValidationOption is a callback to mutate the token prior to use
type ValidationOption func(*Info)
// WithClientCertificate configures certs and keys to be used
// to authenticate the request.
func WithClientCertificate(certFile, keyFile string) ValidationOption {
return func(i *Info) {
i.CertFile = certFile
i.KeyFile = keyFile
}
}
// WithUser overrides the username from the token with the provided value.
func WithUser(username string) ValidationOption {
return func(i *Info) {
i.Username = username
}
}
// String returns the token data in K10 format
func (i *Info) String() string {
creds := i.Username + ":" + i.Password
if i.BootstrapTokenString != nil {
creds = i.BootstrapTokenString.String()
}
digest, _ := hashCA(i.CACerts)
return fmt.Sprintf(tokenFormat, tokenPrefix, digest, i.Username, i.Password)
return tokenPrefix + digest + "::" + creds
}
// Token returns the bootstrap token string, if available.
func (i *Info) Token() string {
if i.BootstrapTokenString != nil {
return i.BootstrapTokenString.String()
}
return ""
}
// ParseAndValidateToken parses a token, downloads and validates the server's CA bundle,
// and validates it according to the caHash from the token if set.
func ParseAndValidateToken(server string, token string) (*Info, error) {
func ParseAndValidateToken(server string, token string, options ...ValidationOption) (*Info, error) {
info, err := parseToken(token)
if err != nil {
return nil, err
}
if err := info.setAndValidateServer(server); err != nil {
return nil, err
for _, option := range options {
option(info)
}
return info, nil
}
// ParseAndValidateTokenForUser parses a token with user override, downloads and
// validates the server's CA bundle, and validates it according to the caHash from the token if set.
func ParseAndValidateTokenForUser(server, token, username string) (*Info, error) {
info, err := parseToken(token)
if err != nil {
return nil, err
}
info.Username = username
if err := info.setAndValidateServer(server); err != nil {
return nil, err
}
@ -159,13 +181,21 @@ func parseToken(token string) (*Info, error) {
return nil, errors.New("token must not be empty")
}
// Turn bare password or bootstrap token into full K10 token with empty CA hash,
// for consistent parsing in the section below.
if !strings.HasPrefix(token, tokenPrefix) {
token = fmt.Sprintf(tokenFormat, tokenPrefix, "", "", token)
_, err := kubeadm.NewBootstrapTokenString(token)
if err != nil {
token = tokenPrefix + ":::" + token
} else {
token = tokenPrefix + "::" + token
}
}
// Strip off the prefix
// Strip off the prefix.
token = token[len(tokenPrefix):]
// Split into CA hash and creds.
parts := strings.SplitN(token, "::", 2)
token = parts[0]
if len(parts) > 1 {
@ -177,14 +207,20 @@ func parseToken(token string) (*Info, error) {
token = parts[1]
}
parts = strings.SplitN(token, ":", 2)
if len(parts) != 2 || len(parts[1]) == 0 {
return nil, errors.New("invalid token format")
// Try to parse creds as bootstrap token string; fall back to basic auth.
// If neither works, error.
bts, err := kubeadm.NewBootstrapTokenString(token)
if err != nil {
parts = strings.SplitN(token, ":", 2)
if len(parts) != 2 || len(parts[1]) == 0 {
return nil, errors.New("invalid token format")
}
info.Username = parts[0]
info.Password = parts[1]
} else {
info.BootstrapTokenString = bts
}
info.Username = parts[0]
info.Password = parts[1]
return &info, nil
}
@ -192,21 +228,30 @@ func parseToken(token string) (*Info, error) {
// If the CA bundle is empty, it validates using the default http client using the OS CA bundle.
// If the CA bundle is not empty but does not contain any valid certs, it validates using
// an empty CA bundle (which will always fail).
func GetHTTPClient(cacerts []byte) *http.Client {
// If valid cert+key paths can be loaded from the provided paths, they are used for client cert auth.
func GetHTTPClient(cacerts []byte, certFile, keyFile string) *http.Client {
if len(cacerts) == 0 {
return defaultClient
}
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(cacerts)
tlsConfig := &tls.Config{
RootCAs: x509.NewCertPool(),
}
tlsConfig.RootCAs.AppendCertsFromPEM(cacerts)
// Try to load certs from the provided cert and key. We ignore errors,
// as it is OK if the paths were empty or the files don't currently exist.
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err == nil {
tlsConfig.Certificates = []tls.Certificate{cert}
}
return &http.Client{
Timeout: defaultClientTimeout,
Transport: &http.Transport{
DisableKeepAlives: true,
TLSClientConfig: &tls.Config{
RootCAs: pool,
},
TLSClientConfig: tlsConfig,
},
}
}
@ -223,7 +268,7 @@ func (i *Info) Get(path string) ([]byte, error) {
}
p.Scheme = u.Scheme
p.Host = u.Host
return get(p.String(), GetHTTPClient(i.CACerts), i.Username, i.Password)
return get(p.String(), GetHTTPClient(i.CACerts, i.CertFile, i.KeyFile), i.Username, i.Password, i.Token())
}
// Put makes a request to a subpath of info's BaseURL
@ -238,7 +283,7 @@ func (i *Info) Put(path string, body []byte) error {
}
p.Scheme = u.Scheme
p.Host = u.Host
return put(p.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password)
return put(p.String(), body, GetHTTPClient(i.CACerts, i.CertFile, i.KeyFile), i.Username, i.Password, i.Token())
}
// setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
@ -296,13 +341,13 @@ func getCACerts(u url.URL) ([]byte, error) {
// This first request is expected to fail. If the server has
// a cert that can be validated using the default CA bundle, return
// success with no CA certs.
_, err := get(url, defaultClient, "", "")
_, err := get(url, defaultClient, "", "", "")
if err == nil {
return nil, nil
}
// Download the CA bundle using a client that does not validate certs.
cacerts, err := get(url, insecureClient, "", "")
cacerts, err := get(url, insecureClient, "", "", "")
if err != nil {
return nil, errors.Wrap(err, "failed to get CA certs")
}
@ -310,7 +355,7 @@ func getCACerts(u url.URL) ([]byte, error) {
// Request the CA bundle again, validating that the CA bundle can be loaded
// and used to validate the server certificate. This should only fail if we somehow
// get an empty CA bundle. or if the dynamiclistener cert is incorrectly signed.
_, err = get(url, GetHTTPClient(cacerts), "", "")
_, err = get(url, GetHTTPClient(cacerts, "", ""), "", "", "")
if err != nil {
return nil, errors.Wrap(err, "CA cert validation failed")
}
@ -320,13 +365,15 @@ func getCACerts(u url.URL) ([]byte, error) {
// get makes a request to a url using a provided client, username, and password,
// returning the response body.
func get(u string, client *http.Client, username, password string) ([]byte, error) {
func get(u string, client *http.Client, username, password, token string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, u, nil)
if err != nil {
return nil, err
}
if username != "" {
if token != "" {
req.Header.Add("Authorization", "Bearer "+token)
} else if username != "" {
req.SetBasicAuth(username, password)
}
@ -345,13 +392,15 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro
// put makes a request to a url using a provided client, username, and password
// only an error is returned
func put(u string, body []byte, client *http.Client, username, password string) error {
func put(u string, body []byte, client *http.Client, username, password, token string) error {
req, err := http.NewRequest(http.MethodPut, u, bytes.NewBuffer(body))
if err != nil {
return err
}
if username != "" {
if token != "" {
req.Header.Add("Authorization", "Bearer "+token)
} else if username != "" {
req.SetBasicAuth(username, password)
}

View File

@ -21,6 +21,7 @@ import (
var (
defaultUsername = "server"
defaultPassword = "token"
defaultToken = "abcdef.0123456789abcdef"
)
// Test_UnitTrustedCA confirms that tokens are validated when the server uses a cert (self-signed or otherwise)
@ -62,7 +63,7 @@ func Test_UnitTrustedCA(t *testing.T) {
assert.Equal(testCase.expected, info.Username, testCase.token)
}
info, err = ParseAndValidateTokenForUser(server.URL, testCase.token, "agent")
info, err = ParseAndValidateToken(server.URL, testCase.token, WithUser("agent"))
if assert.NoError(err, testCase) {
assert.Nil(info.CACerts, testCase)
assert.Equal("agent", info.Username, testCase)
@ -108,7 +109,7 @@ func Test_UnitUntrustedCA(t *testing.T) {
assert.Equal(testCase.expected, info.Username, testCase)
}
info, err = ParseAndValidateTokenForUser(server.URL, testCase.token, "agent")
info, err = ParseAndValidateToken(server.URL, testCase.token, WithUser("agent"))
if assert.NoError(err, testCase) {
assert.Equal(testInfo.CACerts, info.CACerts, testCase)
assert.Equal("agent", info.Username, testCase)
@ -132,11 +133,41 @@ func Test_UnitInvalidServers(t *testing.T) {
_, err := ParseAndValidateToken(testCase.server, testCase.token)
assert.EqualError(err, testCase.expected, testCase)
_, err = ParseAndValidateTokenForUser(testCase.server, testCase.token, defaultUsername)
_, err = ParseAndValidateToken(testCase.server, testCase.token, WithUser(defaultUsername))
assert.EqualError(err, testCase.expected, testCase)
}
}
// Test_UnitValidTokens tests that valid tokens can be parsed, and give the expected result
func Test_UnitValidTokens(t *testing.T) {
assert := assert.New(t)
server := newTLSServer(t, defaultUsername, defaultPassword, false)
defer server.Close()
digest, _ := hashCA(getServerCA(server))
testCases := []struct {
server string
token string
expectUsername string
expectPassword string
expectToken string
}{
{server.URL, defaultPassword, "", defaultPassword, ""},
{server.URL, defaultToken, "", "", defaultToken},
{server.URL, "K10" + digest + ":::" + defaultPassword, "", defaultPassword, ""},
{server.URL, "K10" + digest + "::" + defaultUsername + ":" + defaultPassword, defaultUsername, defaultPassword, ""},
{server.URL, "K10" + digest + "::" + defaultToken, "", "", defaultToken},
}
for _, testCase := range testCases {
info, err := ParseAndValidateToken(testCase.server, testCase.token)
assert.NoError(err)
assert.Equal(testCase.expectUsername, info.Username, testCase)
assert.Equal(testCase.expectPassword, info.Password, testCase)
assert.Equal(testCase.expectToken, info.Token(), testCase)
}
}
// Test_UnitInvalidTokens tests that tokens which are empty, invalid, or incorrect are properly rejected
func Test_UnitInvalidTokens(t *testing.T) {
assert := assert.New(t)
@ -164,7 +195,7 @@ func Test_UnitInvalidTokens(t *testing.T) {
assert.EqualError(err, testCase.expected, testCase)
assert.Nil(info, testCase)
info, err = ParseAndValidateTokenForUser(testCase.server, testCase.token, defaultUsername)
info, err = ParseAndValidateToken(testCase.server, testCase.token, WithUser(defaultUsername))
assert.EqualError(err, testCase.expected, testCase)
assert.Nil(info, testCase)
}
@ -199,7 +230,7 @@ func Test_UnitInvalidCredentials(t *testing.T) {
assert.Empty(res, testCase)
}
info, err = ParseAndValidateTokenForUser(server.URL, testCase, defaultUsername)
info, err = ParseAndValidateToken(server.URL, testCase, WithUser(defaultUsername))
assert.NoError(err, testCase)
if assert.NotNil(info) {
res, err := info.Get("/v1-k3s/server-bootstrap")
@ -219,7 +250,7 @@ func Test_UnitWrongCert(t *testing.T) {
assert.Error(err)
assert.Nil(info)
info, err = ParseAndValidateTokenForUser(server.URL, defaultPassword, defaultUsername)
info, err = ParseAndValidateToken(server.URL, defaultPassword, WithUser(defaultUsername))
assert.Error(err)
assert.Nil(info)
}
@ -244,7 +275,7 @@ func Test_UnitConnectionFailures(t *testing.T) {
assert.WithinDuration(time.Now(), startTime, testDuration, testCase)
startTime = time.Now()
info, err = ParseAndValidateTokenForUser(testCase.server, testCase.token, defaultUsername)
info, err = ParseAndValidateToken(testCase.server, testCase.token, WithUser(defaultUsername))
assert.Error(err, testCase)
assert.Nil(info, testCase)
assert.WithinDuration(startTime, time.Now(), testDuration, testCase)
@ -295,7 +326,7 @@ func Test_UnitParseAndGet(t *testing.T) {
}
for _, testCase := range testCases {
info, err := ParseAndValidateTokenForUser(server.URL+testCase.extraBasePre, defaultPassword, defaultUsername)
info, err := ParseAndValidateToken(server.URL+testCase.extraBasePre, defaultPassword, WithUser(defaultUsername))
// Check for expected error when parsing server + token
if testCase.parseFail {
assert.Error(err, testCase)

View File

@ -94,7 +94,7 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
// etcd is promoted from learner. Odds are we won't need this info, and we don't want to fail startup
// due to failure to retrieve it as this will break cold cluster restart, so we ignore any errors.
if c.config.JoinURL != "" && c.config.Token != "" {
c.clientAccessInfo, _ = clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
c.clientAccessInfo, _ = clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, clientaccess.WithUser("server"))
}
return false, true, nil
} else if c.config.JoinURL == "" {
@ -109,7 +109,7 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
// Fail if the token isn't syntactically valid, or if the CA hash on the remote server doesn't match
// the hash in the token. The password isn't actually checked until later when actually bootstrapping.
info, err := clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, clientaccess.WithUser("server"))
if err != nil {
return false, false, err
}
@ -453,7 +453,7 @@ func (c *Cluster) compareConfig() error {
if token == "" {
token = c.config.Token
}
agentClientAccessInfo, err := clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, token, "node")
agentClientAccessInfo, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token, clientaccess.WithUser("node"))
if err != nil {
return err
}

View File

@ -76,6 +76,8 @@ type Agent struct {
PodManifests string
NodeName string
NodeConfigPath string
ClientKubeletCert string
ClientKubeletKey string
ServingKubeletCert string
ServingKubeletKey string
ServiceCIDR *net.IPNet