diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 554555a5ed..32fa16203a 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -35,7 +35,6 @@ import ( "k8s.io/client-go/kubernetes" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/clientcmd" - "k8s.io/controller-manager/app" app2 "k8s.io/kubernetes/cmd/kube-proxy/app" kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" utilsnet "k8s.io/utils/net" @@ -97,7 +96,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { return err } - app.WaitForAPIServer(coreClient, 30*time.Second) + util.WaitForAPIServerReady(coreClient, 30*time.Second) if !nodeConfig.NoFlannel { if err := flannel.Run(ctx, nodeConfig, coreClient.CoreV1().Nodes()); err != nil { diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 8e8db3c634..7b8bfcc95a 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -16,7 +16,7 @@ import ( "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/daemons/control/deps" "github.com/rancher/k3s/pkg/daemons/executor" - util2 "github.com/rancher/k3s/pkg/util" + "github.com/rancher/k3s/pkg/util" "github.com/rancher/k3s/pkg/version" "github.com/rancher/wrangler/pkg/generated/controllers/rbac" "github.com/sirupsen/logrus" @@ -26,9 +26,8 @@ import ( "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" - app2 "k8s.io/controller-manager/app" "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" - "k8s.io/kubernetes/pkg/proxy/util" + proxyutil "k8s.io/kubernetes/pkg/proxy/util" // for client metric registration _ "k8s.io/component-base/metrics/prometheus/restclient" @@ -47,7 +46,7 @@ func Server(ctx context.Context, cfg *config.Control) error { } cfg.Runtime.Tunnel = setupTunnel() - util.DisableProxyHostnameCheck = true + proxyutil.DisableProxyHostnameCheck = true var auth authenticator.Request var handler http.Handler @@ -96,7 +95,7 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) erro "kubeconfig": runtime.KubeConfigController, "service-account-private-key-file": runtime.ServiceKey, "allocate-node-cidrs": "true", - "cluster-cidr": util2.JoinIPNets(cfg.ClusterIPRanges), + "cluster-cidr": util.JoinIPNets(cfg.ClusterIPRanges), "root-ca-file": runtime.ServerCA, "profiling": "false", "bind-address": localhostIP.String(), @@ -153,7 +152,7 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control argsMap["allow-privileged"] = "true" argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",") argsMap["service-account-signing-key-file"] = runtime.ServiceKey - argsMap["service-cluster-ip-range"] = util2.JoinIPNets(cfg.ServiceIPRanges) + argsMap["service-cluster-ip-range"] = util.JoinIPNets(cfg.ServiceIPRanges) argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String() argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort) if cfg.AdvertiseIP != "" { @@ -286,14 +285,14 @@ func setupStorageBackend(argsMap map[string]string, cfg *config.Control) { func cloudControllerManager(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) error { argsMap := map[string]string{ "profiling": "false", - "allocate-node-cidrs": "true", // ccmOptions.KubeCloudShared.AllocateNodeCIDRs = true - "cloud-provider": version.Program, // ccmOptions.KubeCloudShared.CloudProvider.Name = version.Program - "cluster-cidr": util2.JoinIPNets(cfg.ClusterIPRanges), // ccmOptions.KubeCloudShared.ClusterCIDR = util2.JoinIPNets(cfg.ClusterIPRanges) - "configure-cloud-routes": "false", // ccmOptions.KubeCloudShared.ConfigureCloudRoutes = false - "kubeconfig": runtime.KubeConfigCloudController, // ccmOptions.Kubeconfig = runtime.KubeConfigCloudController - "node-status-update-frequency": "1m0s", // ccmOptions.NodeStatusUpdateFrequency = metav1.Duration{Duration: 1 * time.Minute} - "bind-address": "127.0.0.1", // ccmOptions.SecureServing.BindAddress = localhostIP - "port": "0", // ccmOptions.SecureServing.BindPort = 0 + "allocate-node-cidrs": "true", + "cloud-provider": version.Program, + "cluster-cidr": util.JoinIPNets(cfg.ClusterIPRanges), + "configure-cloud-routes": "false", + "kubeconfig": runtime.KubeConfigCloudController, + "node-status-update-frequency": "1m0s", + "bind-address": "127.0.0.1", + "port": "0", } if cfg.NoLeaderElect { argsMap["leader-elect"] = "false" @@ -394,7 +393,7 @@ func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRu select { case <-ctx.Done(): return - case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 30*time.Second) }): + case err := <-promise(func() error { return util.WaitForAPIServerReady(k8sClient, 30*time.Second) }): if err != nil { logrus.Infof("Waiting for API server to become available") continue diff --git a/pkg/util/api.go b/pkg/util/api.go index 49d4a5d021..03e10adafe 100644 --- a/pkg/util/api.go +++ b/pkg/util/api.go @@ -1,10 +1,17 @@ package util import ( + "context" + "fmt" "net" + "net/http" "strconv" + "time" + "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" ) func GetAddresses(endpoint *v1.Endpoints) []string { @@ -26,3 +33,32 @@ func GetAddresses(endpoint *v1.Endpoints) []string { } return serverAddresses } + +// WaitForAPIServerReady waits for the API Server's /readyz endpoint to report "ok" with timeout. +// This is cribbed from the Kubernetes controller-manager app, but checks the readyz endpoint instead of the deprecated healthz endpoint. +func WaitForAPIServerReady(client clientset.Interface, timeout time.Duration) error { + var lastErr error + + err := wait.PollImmediate(time.Second, timeout, func() (bool, error) { + healthStatus := 0 + result := client.Discovery().RESTClient().Get().AbsPath("/readyz").Do(context.TODO()).StatusCode(&healthStatus) + if result.Error() != nil { + lastErr = fmt.Errorf("failed to get apiserver /readyz status: %v", result.Error()) + return false, nil + } + if healthStatus != http.StatusOK { + content, _ := result.Raw() + lastErr = fmt.Errorf("APIServer isn't ready: %v", string(content)) + logrus.Warnf("APIServer isn't ready yet: %v. Waiting a little while.", string(content)) + return false, nil + } + + return true, nil + }) + + if err != nil { + return fmt.Errorf("%v: %v", err, lastErr) + } + + return nil +}