Start kube-apiserver in the background

In rke2 everything is a static pod so this causes a chicken and egg situation
in which we need the kubelet running before the kube-apiserver can be
launched.  By starting the apiserver in the background this allows us to
do this odd bootstrapping.

pkg/cli/server/server.go

@@ -185,11 +185,14 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 		return err
 	}
 
-	logrus.Info("k3s is up and running")
-	if notifySocket != "" {
-		os.Setenv("NOTIFY_SOCKET", notifySocket)
-		systemd.SdNotify(true, "READY=1\n")
-	}
+	go func() {
+		<-serverConfig.ControlConfig.Runtime.APIServerReady
+		logrus.Info("k3s is up and running")
+		if notifySocket != "" {
+			os.Setenv("NOTIFY_SOCKET", notifySocket)
+			systemd.SdNotify(true, "READY=1\n")
+		}
+	}()
 
 	if cfg.DisableAgent {
 		<-ctx.Done()

pkg/daemons/config/types.go

@@ -141,7 +141,8 @@ type ControlRuntimeBootstrap struct {
 type ControlRuntime struct {
 	ControlRuntimeBootstrap
 
-	HTTPBootstrap bool
+	HTTPBootstrap  bool
+	APIServerReady <-chan struct{}
 
 	ClientKubeAPICert string
 	ClientKubeAPIKey  string

pkg/daemons/control/server.go

@@ -73,7 +73,6 @@ users:
 )
 
 const (
-	userTokenSize  = 8
 	ipsecTokenSize = 48
 	aescbcKeySize  = 32
 )
@@ -96,7 +95,7 @@ func Server(ctx context.Context, cfg *config.Control) error {
 		return err
 	}
 
-	if err := waitForAPIServer(ctx, runtime); err != nil {
+	if err := waitForAPIServerInBackground(ctx, runtime); err != nil {
 		return err
 	}
 
@@ -141,7 +140,7 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) erro
 	args := config.GetArgsList(argsMap, cfg.ExtraControllerArgs)
 	logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
 
-	return executor.ControllerManager(args)
+	return executor.ControllerManager(runtime.APIServerReady, args)
 }
 
 func scheduler(cfg *config.Control, runtime *config.ControlRuntime) error {
@@ -157,7 +156,7 @@ func scheduler(cfg *config.Control, runtime *config.ControlRuntime) error {
 	args := config.GetArgsList(argsMap, cfg.ExtraSchedulerAPIArgs)
 
 	logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
-	return executor.Scheduler(args)
+	return executor.Scheduler(runtime.APIServerReady, args)
 }
 
 func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
@@ -826,7 +825,7 @@ func checkForCloudControllerPrivileges(runtime *config.ControlRuntime) error {
 	return nil
 }
 
-func waitForAPIServer(ctx context.Context, runtime *config.ControlRuntime) error {
+func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRuntime) error {
 	restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
 	if err != nil {
 		return err
@@ -837,12 +836,27 @@ func waitForAPIServer(ctx context.Context, runtime *config.ControlRuntime) error
 		return err
 	}
 
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 5*time.Minute) }):
-		return err
-	}
+	done := make(chan struct{})
+	runtime.APIServerReady = done
+
+	go func() {
+		defer close(done)
+		logrus.Infof("Waiting for API server to become available")
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 30*time.Second) }):
+				if err != nil {
+					logrus.Infof("Waiting for API server to become available")
+					continue
+				}
+				return
+			}
+		}
+	}()
+
+	return nil
 }
 
 func promise(f func() error) <-chan error {

pkg/daemons/executor/embed.go

@@ -57,22 +57,24 @@ func (Embedded) APIServer(ctx context.Context, args []string) (authenticator.Req
 	return startupConfig.Authenticator, startupConfig.Handler, nil
 }
 
-func (Embedded) Scheduler(args []string) error {
+func (Embedded) Scheduler(apiReady <-chan struct{}, args []string) error {
 	command := sapp.NewSchedulerCommand()
 	command.SetArgs(args)
 
 	go func() {
+		<-apiReady
 		logrus.Fatalf("scheduler exited: %v", command.Execute())
 	}()
 
 	return nil
 }
 
-func (Embedded) ControllerManager(args []string) error {
+func (Embedded) ControllerManager(apiReady <-chan struct{}, args []string) error {
 	command := cmapp.NewControllerManagerCommand()
 	command.SetArgs(args)
 
 	go func() {
+		<-apiReady
 		logrus.Fatalf("controller-manager exited: %v", command.Execute())
 	}()
 

pkg/daemons/executor/executor.go

@@ -11,8 +11,8 @@ type Executor interface {
 	Kubelet(args []string) error
 	KubeProxy(args []string) error
 	APIServer(ctx context.Context, args []string) (authenticator.Request, http.Handler, error)
-	Scheduler(args []string) error
-	ControllerManager(args []string) error
+	Scheduler(apiReady <-chan struct{}, args []string) error
+	ControllerManager(apiReady <-chan struct{}, args []string) error
 }
 
 var (
@@ -35,10 +35,10 @@ func APIServer(ctx context.Context, args []string) (authenticator.Request, http.
 	return executor.APIServer(ctx, args)
 }
 
-func Scheduler(args []string) error {
-	return executor.Scheduler(args)
+func Scheduler(apiReady <-chan struct{}, args []string) error {
+	return executor.Scheduler(apiReady, args)
 }
 
-func ControllerManager(args []string) error {
-	return executor.ControllerManager(args)
+func ControllerManager(apiReady <-chan struct{}, args []string) error {
+	return executor.ControllerManager(apiReady, args)
 }

pkg/server/server.go

@@ -87,6 +87,24 @@ func startWrangler(ctx context.Context, config *Config) error {
 
 	controlConfig.Runtime.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, ca)
 
+	// Start in background
+	go func() {
+		select {
+		case <-ctx.Done():
+			return
+		case <-config.ControlConfig.Runtime.APIServerReady:
+			if err := runControllers(ctx, config); err != nil {
+				logrus.Fatal("failed to start controllers: %v", err)
+			}
+		}
+	}()
+
+	return nil
+}
+
+func runControllers(ctx context.Context, config *Config) error {
+	controlConfig := &config.ControlConfig
+
 	sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
 	if err != nil {
 		return err