diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 8fe51e42a1..b09b99f246 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -185,11 +185,14 @@ func run(app *cli.Context, cfg *cmds.Server) error { return err } - logrus.Info("k3s is up and running") - if notifySocket != "" { - os.Setenv("NOTIFY_SOCKET", notifySocket) - systemd.SdNotify(true, "READY=1\n") - } + go func() { + <-serverConfig.ControlConfig.Runtime.APIServerReady + logrus.Info("k3s is up and running") + if notifySocket != "" { + os.Setenv("NOTIFY_SOCKET", notifySocket) + systemd.SdNotify(true, "READY=1\n") + } + }() if cfg.DisableAgent { <-ctx.Done() diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index f4dddb6d95..38dd3eacbb 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -141,7 +141,8 @@ type ControlRuntimeBootstrap struct { type ControlRuntime struct { ControlRuntimeBootstrap - HTTPBootstrap bool + HTTPBootstrap bool + APIServerReady <-chan struct{} ClientKubeAPICert string ClientKubeAPIKey string diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index bd1b743f88..1e934d7f64 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -73,7 +73,6 @@ users: ) const ( - userTokenSize = 8 ipsecTokenSize = 48 aescbcKeySize = 32 ) @@ -96,7 +95,7 @@ func Server(ctx context.Context, cfg *config.Control) error { return err } - if err := waitForAPIServer(ctx, runtime); err != nil { + if err := waitForAPIServerInBackground(ctx, runtime); err != nil { return err } @@ -141,7 +140,7 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) erro args := config.GetArgsList(argsMap, cfg.ExtraControllerArgs) logrus.Infof("Running kube-controller-manager %s", config.ArgString(args)) - return executor.ControllerManager(args) + return executor.ControllerManager(runtime.APIServerReady, args) } func scheduler(cfg *config.Control, runtime *config.ControlRuntime) error { @@ -157,7 +156,7 @@ func scheduler(cfg *config.Control, runtime *config.ControlRuntime) error { args := config.GetArgsList(argsMap, cfg.ExtraSchedulerAPIArgs) logrus.Infof("Running kube-scheduler %s", config.ArgString(args)) - return executor.Scheduler(args) + return executor.Scheduler(runtime.APIServerReady, args) } func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) { @@ -826,7 +825,7 @@ func checkForCloudControllerPrivileges(runtime *config.ControlRuntime) error { return nil } -func waitForAPIServer(ctx context.Context, runtime *config.ControlRuntime) error { +func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRuntime) error { restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin) if err != nil { return err @@ -837,12 +836,27 @@ func waitForAPIServer(ctx context.Context, runtime *config.ControlRuntime) error return err } - select { - case <-ctx.Done(): - return ctx.Err() - case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 5*time.Minute) }): - return err - } + done := make(chan struct{}) + runtime.APIServerReady = done + + go func() { + defer close(done) + logrus.Infof("Waiting for API server to become available") + for { + select { + case <-ctx.Done(): + return + case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 30*time.Second) }): + if err != nil { + logrus.Infof("Waiting for API server to become available") + continue + } + return + } + } + }() + + return nil } func promise(f func() error) <-chan error { diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index d918a54ae8..27bc978484 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -57,22 +57,24 @@ func (Embedded) APIServer(ctx context.Context, args []string) (authenticator.Req return startupConfig.Authenticator, startupConfig.Handler, nil } -func (Embedded) Scheduler(args []string) error { +func (Embedded) Scheduler(apiReady <-chan struct{}, args []string) error { command := sapp.NewSchedulerCommand() command.SetArgs(args) go func() { + <-apiReady logrus.Fatalf("scheduler exited: %v", command.Execute()) }() return nil } -func (Embedded) ControllerManager(args []string) error { +func (Embedded) ControllerManager(apiReady <-chan struct{}, args []string) error { command := cmapp.NewControllerManagerCommand() command.SetArgs(args) go func() { + <-apiReady logrus.Fatalf("controller-manager exited: %v", command.Execute()) }() diff --git a/pkg/daemons/executor/executor.go b/pkg/daemons/executor/executor.go index f7fcc2f2ed..fd19adb43d 100644 --- a/pkg/daemons/executor/executor.go +++ b/pkg/daemons/executor/executor.go @@ -11,8 +11,8 @@ type Executor interface { Kubelet(args []string) error KubeProxy(args []string) error APIServer(ctx context.Context, args []string) (authenticator.Request, http.Handler, error) - Scheduler(args []string) error - ControllerManager(args []string) error + Scheduler(apiReady <-chan struct{}, args []string) error + ControllerManager(apiReady <-chan struct{}, args []string) error } var ( @@ -35,10 +35,10 @@ func APIServer(ctx context.Context, args []string) (authenticator.Request, http. return executor.APIServer(ctx, args) } -func Scheduler(args []string) error { - return executor.Scheduler(args) +func Scheduler(apiReady <-chan struct{}, args []string) error { + return executor.Scheduler(apiReady, args) } -func ControllerManager(args []string) error { - return executor.ControllerManager(args) +func ControllerManager(apiReady <-chan struct{}, args []string) error { + return executor.ControllerManager(apiReady, args) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 1e0f4f4104..08317788d3 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -87,6 +87,24 @@ func startWrangler(ctx context.Context, config *Config) error { controlConfig.Runtime.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, ca) + // Start in background + go func() { + select { + case <-ctx.Done(): + return + case <-config.ControlConfig.Runtime.APIServerReady: + if err := runControllers(ctx, config); err != nil { + logrus.Fatal("failed to start controllers: %v", err) + } + } + }() + + return nil +} + +func runControllers(ctx context.Context, config *Config) error { + controlConfig := &config.ControlConfig + sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin) if err != nil { return err