diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 1dba940439..4ca106f433 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -82,7 +82,7 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester { defer resp.Body.Close() if resp.StatusCode == http.StatusForbidden { - return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile) + return nil, fmt.Errorf("Node password rejected, duplicate hostname or contents of '%s' may not match server node-passwd entry, try enabling a unique node name with the --with-node-id flag", nodePasswordFile) } if resp.StatusCode != http.StatusOK { @@ -93,6 +93,20 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester { } } +func ensureNodeID(nodeIDFile string) (string, error) { + if _, err := os.Stat(nodeIDFile); err == nil { + id, err := ioutil.ReadFile(nodeIDFile) + return strings.TrimSpace(string(id)), err + } + id := make([]byte, 4, 4) + _, err := cryptorand.Read(id) + if err != nil { + return "", err + } + nodeID := hex.EncodeToString(id) + return nodeID, ioutil.WriteFile(nodeIDFile, []byte(nodeID+"\n"), 0644) +} + func ensureNodePassword(nodePasswordFile string) (string, error) { if _, err := os.Stat(nodePasswordFile); err == nil { password, err := ioutil.ReadFile(nodePasswordFile) @@ -107,6 +121,21 @@ func ensureNodePassword(nodePasswordFile string) (string, error) { return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600) } +func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string) { + password, err := ioutil.ReadFile(oldNodePasswordFile) + if err != nil { + return + } + if err := ioutil.WriteFile(newNodePasswordFile, password, 0600); err != nil { + logrus.Warnf("Unable to write password file: %v", err) + return + } + if err := os.Remove(oldNodePasswordFile); err != nil { + logrus.Warnf("Unable to remove old password file: %v", err) + return + } +} + func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) { servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile)) if err != nil { @@ -244,11 +273,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - nodeName, nodeIP, err := getHostnameAndIP(*envInfo) - if err != nil { - return nil, err - } - hostLocal, err := exec.LookPath("host-local") if err != nil { return nil, errors.Wrapf(err, "failed to find host-local") @@ -274,14 +298,40 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt") servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key") - nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt") - servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info) + + nodePasswordRoot := "/" + if envInfo.Rootless { + nodePasswordRoot = envInfo.DataDir + } + nodeConfigPath := filepath.Join(nodePasswordRoot, "etc", "rancher", "node") + if err := os.MkdirAll(nodeConfigPath, 0755); err != nil { + return nil, err + } + + oldNodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt") + newNodePasswordFile := filepath.Join(nodeConfigPath, "password") + upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile) + + nodeName, nodeIP, err := getHostnameAndIP(*envInfo) + if err != nil { + return nil, err + } + + if envInfo.WithNodeID { + nodeID, err := ensureNodeID(filepath.Join(nodeConfigPath, "id")) + if err != nil { + return nil, err + } + nodeName += "-" + nodeID + } + + servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info) if err != nil { return nil, err } clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt") - if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil { + if err := getNodeNamedHostFile(clientKubeletCert, nodeName, newNodePasswordFile, info); err != nil { return nil, err } @@ -334,6 +384,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.Images = filepath.Join(envInfo.DataDir, "images") nodeConfig.AgentConfig.NodeIP = nodeIP nodeConfig.AgentConfig.NodeName = nodeName + nodeConfig.AgentConfig.NodeConfigPath = nodeConfigPath nodeConfig.AgentConfig.NodeExternalIP = envInfo.NodeExternalIP nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 8f43666476..d66c51b8a2 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -26,6 +26,7 @@ type Agent struct { Debug bool Rootless bool RootlessAlreadyUnshared bool + WithNodeID bool AgentShared ExtraKubeletArgs cli.StringSlice ExtraKubeProxyArgs cli.StringSlice @@ -57,6 +58,11 @@ var ( EnvVar: "K3S_NODE_NAME", Destination: &AgentConfig.NodeName, } + WithNodeIDFlag = cli.BoolFlag{ + Name: "with-node-id", + Usage: "(agent/node) Append id to node name", + Destination: &AgentConfig.WithNodeID, + } DockerFlag = cli.BoolFlag{ Name: "docker", Usage: "(agent/runtime) Use docker instead of containerd", diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index fa63fff7a0..edcfdcf5de 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -210,6 +210,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.DisableNPC, }, NodeNameFlag, + WithNodeIDFlag, NodeLabels, NodeTaints, DockerFlag, diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index d4b38aa4cb..165616ed21 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -14,8 +14,8 @@ import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/net" "k8s.io/component-base/logs" - app2 "k8s.io/kubernetes/cmd/kube-proxy/app" - "k8s.io/kubernetes/cmd/kubelet/app" + proxy "k8s.io/kubernetes/cmd/kube-proxy/app" + kubelet "k8s.io/kubernetes/cmd/kubelet/app" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration @@ -25,34 +25,37 @@ import ( func Agent(config *config.Agent) error { rand.Seed(time.Now().UTC().UnixNano()) - kubelet(config) - kubeProxy(config) + logs.InitLogs() + defer logs.FlushLogs() + + startKubelet(config) + startKubeProxy(config) return nil } -func kubeProxy(cfg *config.Agent) { +func startKubeProxy(cfg *config.Agent) { argsMap := map[string]string{ "proxy-mode": "iptables", "healthz-bind-address": "127.0.0.1", "kubeconfig": cfg.KubeConfigKubeProxy, "cluster-cidr": cfg.ClusterCIDR.String(), } - args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs) + if cfg.NodeName != "" { + argsMap["hostname-override"] = cfg.NodeName + } - command := app2.NewProxyCommand() + args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs) + command := proxy.NewProxyCommand() command.SetArgs(args) + go func() { - err := command.Execute() - logrus.Fatalf("kube-proxy exited: %v", err) + logrus.Infof("Running kube-proxy %s", config.ArgString(args)) + logrus.Fatalf("kube-proxy exited: %v", command.Execute()) }() } -func kubelet(cfg *config.Agent) { - command := app.NewKubeletCommand(context.Background().Done()) - logs.InitLogs() - defer logs.FlushLogs() - +func startKubelet(cfg *config.Agent) { argsMap := map[string]string{ "healthz-bind-address": "127.0.0.1", "read-only-port": "0", @@ -146,6 +149,7 @@ func kubelet(cfg *config.Agent) { } args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs) + command := kubelet.NewKubeletCommand(context.Background().Done()) command.SetArgs(args) go func() { diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index c3750ffa84..cac96859e6 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -48,6 +48,7 @@ type Containerd struct { type Agent struct { NodeName string + NodeConfigPath string ServingKubeletCert string ServingKubeletKey string ClusterCIDR net.IPNet