Merge pull request #2744 from erikwilson/rke2-node-password-bootstrap

Bootstrap node password with local file

pkg/agent/config/config.go

@@ -361,6 +361,8 @@ func get(envInfo *cmds.Agent, proxy proxy.Proxy) (*config.Node, error) {
 		nodeName += "-" + nodeID
 	}
 
+	os.Setenv("NODE_NAME", nodeName)
+
 	servingCert, err := getServingCert(nodeName, nodeIP, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info)
 	if err != nil {
 		return nil, err
@@ -485,8 +487,6 @@ func get(envInfo *cmds.Agent, proxy proxy.Proxy) (*config.Node, error) {
 		nodeConfig.AgentConfig.ClusterCIDR = *controlConfig.ClusterIPRange
 	}
 
-	os.Setenv("NODE_NAME", nodeConfig.AgentConfig.NodeName)
-
 	nodeConfig.AgentConfig.ExtraKubeletArgs = envInfo.ExtraKubeletArgs
 	nodeConfig.AgentConfig.ExtraKubeProxyArgs = envInfo.ExtraKubeProxyArgs
 

pkg/server/router.go

@@ -1,17 +1,21 @@
 package server
 
 import (
+	"context"
 	"crypto"
 	"crypto/x509"
-	"errors"
 	"io/ioutil"
 	"net"
 	"net/http"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/k3s/pkg/bootstrap"
 	"github.com/rancher/k3s/pkg/daemons/config"
@@ -26,13 +30,16 @@ const (
 	staticURL = "/static/"
 )
 
-func router(serverConfig *config.Control) http.Handler {
+func router(ctx context.Context, config *Config) http.Handler {
+	serverConfig := &config.ControlConfig
+	nodeAuth := passwordBootstrap(ctx, config)
+
 	prefix := "/v1-" + version.Program
 	authed := mux.NewRouter()
 	authed.Use(authMiddleware(serverConfig, version.Program+":agent"))
 	authed.NotFoundHandler = serverConfig.Runtime.Handler
-	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, serverConfig.Runtime))
+	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, nodeAuth))
-	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, serverConfig.Runtime))
+	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, nodeAuth))
 	authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey))
 	authed.Path(prefix + "/client-" + version.Program + "-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert, serverConfig.Runtime.ClientK3sControllerKey))
 	authed.Path(prefix + "/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
@@ -126,30 +133,16 @@ func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Cer
 	return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
 }
 
-func servingKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler {
+func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler {
-	var secretClient coreclient.SecretClient
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if secretClient == nil {
-			if runtime.Core == nil {
-				sendError(errors.New("runtime core not ready"), resp)
-				return
-			}
-			secretClient = runtime.Core.Core().V1().Secret()
-		}
-
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
 
-		nodeName, nodePassword, err := getNodeInfo(req)
+		nodeName, errCode, err := auth(req)
 		if err != nil {
-			sendError(err, resp)
+			sendError(err, resp, errCode)
-			return
-		}
-
-		if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil {
-			sendError(err, resp, http.StatusForbidden)
 			return
 		}
 
@@ -188,30 +181,16 @@ func servingKubeletCert(server *config.Control, keyFile string, runtime *config.
 	})
 }
 
-func clientKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler {
+func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler {
-	var secretClient coreclient.SecretClient
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if secretClient == nil {
-			if runtime.Core == nil {
-				sendError(errors.New("runtime core not ready"), resp)
-				return
-			}
-			secretClient = runtime.Core.Core().V1().Secret()
-		}
-
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
 
-		nodeName, nodePassword, err := getNodeInfo(req)
+		nodeName, errCode, err := auth(req)
 		if err != nil {
-			sendError(err, resp)
+			sendError(err, resp, errCode)
-			return
-		}
-
-		if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil {
-			sendError(err, resp, http.StatusForbidden)
 			return
 		}
 
@@ -292,12 +271,95 @@ func serveStatic(urlPrefix, staticDir string) http.Handler {
 }
 
 func sendError(err error, resp http.ResponseWriter, status ...int) {
-	code := http.StatusInternalServerError
+	var code int
 	if len(status) == 1 {
 		code = status[0]
 	}
-
+	if code == 0 || code == http.StatusOK {
+		code = http.StatusInternalServerError
+	}
 	logrus.Error(err)
 	resp.WriteHeader(code)
 	resp.Write([]byte(err.Error()))
 }
+
+// nodePassBootstrapper returns a node name, or http error code and error
+type nodePassBootstrapper func(req *http.Request) (string, int, error)
+
+func passwordBootstrap(ctx context.Context, config *Config) nodePassBootstrapper {
+	runtime := config.ControlConfig.Runtime
+	var secretClient coreclient.SecretClient
+	var once sync.Once
+
+	return nodePassBootstrapper(func(req *http.Request) (string, int, error) {
+		nodeName, nodePassword, err := getNodeInfo(req)
+		if err != nil {
+			return "", http.StatusBadRequest, err
+		}
+
+		if secretClient == nil {
+			if runtime.Core != nil {
+				// initialize the client if we can
+				secretClient = runtime.Core.Core().V1().Secret()
+			} else if nodeName == os.Getenv("NODE_NAME") {
+				// or verify the password locally and ensure a secret later
+				return verifyLocalPassword(ctx, config, &once, nodeName, nodePassword)
+			} else {
+				// or reject the request until the core is ready
+				return "", http.StatusServiceUnavailable, errors.New("runtime core not ready")
+			}
+		}
+
+		if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil {
+			return "", http.StatusForbidden, err
+		}
+
+		return nodeName, http.StatusOK, nil
+	})
+}
+
+func verifyLocalPassword(ctx context.Context, config *Config, once *sync.Once, nodeName, nodePassword string) (string, int, error) {
+	// use same password file location that the agent creates
+	nodePasswordRoot := "/"
+	if config.Rootless {
+		nodePasswordRoot = filepath.Join(config.ControlConfig.DataDir, "agent")
+	}
+	nodeConfigPath := filepath.Join(nodePasswordRoot, "etc", "rancher", "node")
+	nodePasswordFile := filepath.Join(nodeConfigPath, "password")
+
+	passBytes, err := ioutil.ReadFile(nodePasswordFile)
+	if err != nil {
+		return "", http.StatusInternalServerError, errors.Wrap(err, "unable to read node password file")
+	}
+
+	password := strings.TrimSpace(string(passBytes))
+	if password != nodePassword {
+		return "", http.StatusForbidden, errors.Wrapf(err, "unable to verify local password for node '%s'", nodeName)
+	}
+
+	// make sure the secret is created when the api server is ready
+	ensureSecret := func() {
+		runtime := config.ControlConfig.Runtime
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(1 * time.Second):
+				if runtime.Core != nil {
+					logrus.Debugf("runtime core has become available, ensuring password secret for node '%s'", nodeName)
+					secretClient := runtime.Core.Core().V1().Secret()
+					if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil {
+						logrus.Warnf("error ensuring node password secret for pre-validated node '%s': %v", nodeName, err)
+					}
+					return
+				}
+			}
+		}
+	}
+
+	go once.Do(ensureSecret)
+
+	logrus.Debugf("password verified locally for node '%s'", nodeName)
+
+	return nodeName, http.StatusOK, nil
+}

pkg/server/server.go

@@ -60,7 +60,7 @@ func StartServer(ctx context.Context, config *Config) error {
 		return errors.Wrap(err, "starting kubernetes")
 	}
 
-	config.ControlConfig.Runtime.Handler = router(&config.ControlConfig)
+	config.ControlConfig.Runtime.Handler = router(ctx, config)
 
 	go startOnAPIServerReady(ctx, config)