diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 7525198893..c7322eaab7 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -361,6 +361,8 @@ func get(envInfo *cmds.Agent, proxy proxy.Proxy) (*config.Node, error) { nodeName += "-" + nodeID } + os.Setenv("NODE_NAME", nodeName) + servingCert, err := getServingCert(nodeName, nodeIP, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info) if err != nil { return nil, err @@ -485,8 +487,6 @@ func get(envInfo *cmds.Agent, proxy proxy.Proxy) (*config.Node, error) { nodeConfig.AgentConfig.ClusterCIDR = *controlConfig.ClusterIPRange } - os.Setenv("NODE_NAME", nodeConfig.AgentConfig.NodeName) - nodeConfig.AgentConfig.ExtraKubeletArgs = envInfo.ExtraKubeletArgs nodeConfig.AgentConfig.ExtraKubeProxyArgs = envInfo.ExtraKubeProxyArgs diff --git a/pkg/server/router.go b/pkg/server/router.go index 2d5a6a68a5..88406d3e1d 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -1,17 +1,21 @@ package server import ( + "context" "crypto" "crypto/x509" - "errors" "io/ioutil" "net" "net/http" + "os" "path/filepath" "strconv" "strings" + "sync" + "time" "github.com/gorilla/mux" + "github.com/pkg/errors" certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/k3s/pkg/bootstrap" "github.com/rancher/k3s/pkg/daemons/config" @@ -26,13 +30,16 @@ const ( staticURL = "/static/" ) -func router(serverConfig *config.Control) http.Handler { +func router(ctx context.Context, config *Config) http.Handler { + serverConfig := &config.ControlConfig + nodeAuth := passwordBootstrap(ctx, config) + prefix := "/v1-" + version.Program authed := mux.NewRouter() authed.Use(authMiddleware(serverConfig, version.Program+":agent")) authed.NotFoundHandler = serverConfig.Runtime.Handler - authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, serverConfig.Runtime)) - authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, serverConfig.Runtime)) + authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, nodeAuth)) + authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, nodeAuth)) authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey)) authed.Path(prefix + "/client-" + version.Program + "-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert, serverConfig.Runtime.ClientK3sControllerKey)) authed.Path(prefix + "/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA)) @@ -126,30 +133,16 @@ func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Cer return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil } -func servingKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler { - var secretClient coreclient.SecretClient +func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if secretClient == nil { - if runtime.Core == nil { - sendError(errors.New("runtime core not ready"), resp) - return - } - secretClient = runtime.Core.Core().V1().Secret() - } - if req.TLS == nil { resp.WriteHeader(http.StatusNotFound) return } - nodeName, nodePassword, err := getNodeInfo(req) - if err != nil { - sendError(err, resp) - return - } - - if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil { - sendError(err, resp, http.StatusForbidden) + nodeName, code, err := auth(req) + if err != nil || code != 0 { + sendError(err, resp, code) return } @@ -188,30 +181,16 @@ func servingKubeletCert(server *config.Control, keyFile string, runtime *config. }) } -func clientKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler { - var secretClient coreclient.SecretClient +func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBootstrapper) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - if secretClient == nil { - if runtime.Core == nil { - sendError(errors.New("runtime core not ready"), resp) - return - } - secretClient = runtime.Core.Core().V1().Secret() - } - if req.TLS == nil { resp.WriteHeader(http.StatusNotFound) return } - nodeName, nodePassword, err := getNodeInfo(req) - if err != nil { - sendError(err, resp) - return - } - - if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil { - sendError(err, resp, http.StatusForbidden) + nodeName, code, err := auth(req) + if err != nil || code != 0 { + sendError(err, resp, code) return } @@ -293,11 +272,94 @@ func serveStatic(urlPrefix, staticDir string) http.Handler { func sendError(err error, resp http.ResponseWriter, status ...int) { code := http.StatusInternalServerError - if len(status) == 1 { + if len(status) == 1 && status[0] != 0 { code = status[0] } - + if err == nil { + err = errors.New("unknown") + } logrus.Error(err) resp.WriteHeader(code) resp.Write([]byte(err.Error())) } + +// nodePassBootstrapper returns a node name, http code, and error +type nodePassBootstrapper func(req *http.Request) (string, int, error) + +func passwordBootstrap(ctx context.Context, config *Config) nodePassBootstrapper { + runtime := config.ControlConfig.Runtime + var secretClient coreclient.SecretClient + var once sync.Once + + return nodePassBootstrapper(func(req *http.Request) (string, int, error) { + nodeName, nodePassword, err := getNodeInfo(req) + if err != nil { + return "", http.StatusBadRequest, err + } + + if secretClient == nil { + if runtime.Core != nil { + // initialize the client if we can + secretClient = runtime.Core.Core().V1().Secret() + } else if nodeName == os.Getenv("NODE_NAME") { + // or verify the password locally and ensure a secret later + return verifyLocalPassword(ctx, config, &once, nodeName, nodePassword) + } else { + // or reject the request until the core is ready + return "", http.StatusServiceUnavailable, errors.New("runtime core not ready") + } + } + + if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil { + return "", http.StatusForbidden, err + } + + return nodeName, 0, nil + }) +} + +func verifyLocalPassword(ctx context.Context, config *Config, once *sync.Once, nodeName, nodePassword string) (string, int, error) { + // use same password file location that the agent creates + nodePasswordRoot := "/" + if config.Rootless { + nodePasswordRoot = filepath.Join(config.ControlConfig.DataDir, "agent") + } + nodeConfigPath := filepath.Join(nodePasswordRoot, "etc", "rancher", "node") + nodePasswordFile := filepath.Join(nodeConfigPath, "password") + + passBytes, err := ioutil.ReadFile(nodePasswordFile) + if err != nil { + return "", http.StatusInternalServerError, errors.Wrap(err, "unabled to read node password file") + } + + password := strings.TrimSpace(string(passBytes)) + if password != nodePassword { + return "", http.StatusForbidden, errors.Wrapf(err, "unable to verify local password for node '%s'", nodeName) + } + + // make sure the secret is created when the api server is ready + ensureSecret := func() { + runtime := config.ControlConfig.Runtime + for { + select { + case <-ctx.Done(): + return + case <-time.After(1 * time.Second): + if runtime.Core != nil { + logrus.Debugf("runtime core has become available, ensuring password secret for node '%s'", nodeName) + secretClient := runtime.Core.Core().V1().Secret() + if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil { + logrus.Warnf("error ensuring node password secret for pre-validated node '%s': %v", nodeName, err) + } + return + } + } + } + } + + go once.Do(ensureSecret) + + logrus.Debugf("password verified locally for node '%s'", nodeName) + + return nodeName, 0, nil +} diff --git a/pkg/server/server.go b/pkg/server/server.go index bf31470caf..26adfd58a0 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -60,7 +60,7 @@ func StartServer(ctx context.Context, config *Config) error { return errors.Wrap(err, "starting kubernetes") } - config.ControlConfig.Runtime.Handler = router(&config.ControlConfig) + config.ControlConfig.Runtime.Handler = router(ctx, config) go startOnAPIServerReady(ctx, config)