Mirror
/
k3s
mirror of https://github.com/k3s-io/k3s.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Delay service readiness until after startuphooks have finished (#5736) 

* Move startup hooks wg into a runtime pointer, check before notifying systemd
* Switch default systemd notification to server
* Add 1 sec delay to allow etcd to write to disk
Signed-off-by: Derek Nola <derek.nola@suse.com>
This commit is contained in:
Derek Nola 2022-06-16 08:22:42 -07:00 committed by GitHub
parent dbb0811766
commit ccb09f627a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/agent/run.go
View File

@ -29,6 +29,7 @@ import (
"github.com/rancher/k3s/pkg/nodeconfig"
"github.com/rancher/k3s/pkg/rootless"
"github.com/rancher/k3s/pkg/util"
"github.com/rancher/k3s/pkg/version"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
@ -128,8 +129,13 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
}
}
os.Setenv("NOTIFY_SOCKET", notifySocket)
systemd.SdNotify(true, "READY=1\n")
// By default, the server is responsible for notifying systemd
// On agent-only nodes, the agent will notify systemd
if notifySocket != "" {
logrus.Info(version.Program + " agent is up and running")
os.Setenv("NOTIFY_SOCKET", notifySocket)
systemd.SdNotify(true, "READY=1\n")
}
<-ctx.Done()
return ctx.Err()

11
pkg/cli/server/server.go
View File

@ -419,6 +419,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
logrus.Info("Starting " + version.Program + " " + app.App.Version)
notifySocket := os.Getenv("NOTIFY_SOCKET")
os.Unsetenv("NOTIFY_SOCKET")
ctx := signals.SetupSignalContext()
@ -430,16 +431,16 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
if !serverConfig.ControlConfig.DisableAPIServer {
<-serverConfig.ControlConfig.Runtime.APIServerReady
logrus.Info("Kube API server is now running")
} else {
serverConfig.ControlConfig.Runtime.StartupHooksWg.Wait()
}
if !serverConfig.ControlConfig.DisableETCD {
<-serverConfig.ControlConfig.Runtime.ETCDReady
logrus.Info("ETCD server is now running")
}
logrus.Info(version.Program + " is up and running")
if (cfg.DisableAgent || cfg.DisableAPIServer) && notifySocket != "" {
os.Setenv("NOTIFY_SOCKET", notifySocket)
systemd.SdNotify(true, "READY=1\n")
}
os.Setenv("NOTIFY_SOCKET", notifySocket)
systemd.SdNotify(true, "READY=1\n")
}()
ip := serverConfig.ControlConfig.BindAddress

2
pkg/daemons/config/types.go
View File

@ -8,6 +8,7 @@ import (
"net/http"
"sort"
"strings"
"sync"
"time"
"github.com/k3s-io/kine/pkg/endpoint"
@ -216,6 +217,7 @@ type ControlRuntime struct {
APIServerReady <-chan struct{}
AgentReady <-chan struct{}
ETCDReady <-chan struct{}
StartupHooksWg *sync.WaitGroup
ClusterControllerStart func(ctx context.Context) error
LeaderElectedClusterControllerStart func(ctx context.Context) error

4
pkg/server/secrets-encrypt.go
View File

@ -186,6 +186,10 @@ func encryptionConfigHandler(ctx context.Context, server *config.Control) http.H
genErrorMessage(resp, http.StatusBadRequest, err)
return
}
// If a user kills the k3s server immediately after this call, we run into issues where the files
// have not yet been written. This sleep ensures that things have time to sync to disk before
// the request completes.
time.Sleep(1 * time.Second)
resp.WriteHeader(http.StatusOK)
})
}

12
pkg/server/server.go
View File

@ -66,6 +66,8 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
wg.Add(len(config.StartupHooks))
config.ControlConfig.Runtime.Handler = router(ctx, config, cfg)
config.ControlConfig.Runtime.StartupHooksWg = wg
shArgs := cmds.StartupHookArgs{
APIServerReady: config.ControlConfig.Runtime.APIServerReady,
KubeConfigAdmin: config.ControlConfig.Runtime.KubeConfigAdmin,
@ -81,7 +83,7 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
if config.ControlConfig.DisableAPIServer {
go setETCDLabelsAndAnnotations(ctx, config)
} else {
go startOnAPIServerReady(ctx, wg, config)
go startOnAPIServerReady(ctx, config)
}
ip := net2.ParseIP(config.ControlConfig.BindAddress)
@ -101,18 +103,18 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
return writeKubeConfig(config.ControlConfig.Runtime.ServerCA, config)
}
func startOnAPIServerReady(ctx context.Context, wg *sync.WaitGroup, config *Config) {
func startOnAPIServerReady(ctx context.Context, config *Config) {
select {
case <-ctx.Done():
return
case <-config.ControlConfig.Runtime.APIServerReady:
if err := runControllers(ctx, wg, config); err != nil {
if err := runControllers(ctx, config); err != nil {
logrus.Fatalf("failed to start controllers: %v", err)
}
}
}
func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) error {
func runControllers(ctx context.Context, config *Config) error {
controlConfig := &config.ControlConfig
sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
@ -120,7 +122,7 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err
return err
}
wg.Wait()
controlConfig.Runtime.StartupHooksWg.Wait()
if err := stageFiles(ctx, sc, controlConfig); err != nil {
return err
}