mirror of
https://github.com/k3s-io/k3s.git
synced 2024-06-07 19:41:36 +00:00
fc19b805d5
Updated the logic to handle if extra args are passed with existing hyphens in the arg. The test was updated to add the additional case of having pre-existing hyphens. The method name was also refactored based on previous feedback.
422 lines
13 KiB
Go
422 lines
13 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"math/rand"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/rancher/k3s/pkg/cluster"
|
|
"github.com/rancher/k3s/pkg/daemons/config"
|
|
"github.com/rancher/k3s/pkg/daemons/control/deps"
|
|
"github.com/rancher/k3s/pkg/daemons/executor"
|
|
util2 "github.com/rancher/k3s/pkg/util"
|
|
"github.com/rancher/k3s/pkg/version"
|
|
"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
|
|
"github.com/sirupsen/logrus"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
"k8s.io/apiserver/pkg/authentication/authenticator"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/tools/clientcmd"
|
|
app2 "k8s.io/controller-manager/app"
|
|
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
|
"k8s.io/kubernetes/pkg/proxy/util"
|
|
|
|
// for client metric registration
|
|
_ "k8s.io/component-base/metrics/prometheus/restclient"
|
|
)
|
|
|
|
var localhostIP = net.ParseIP("127.0.0.1")
|
|
|
|
func Server(ctx context.Context, cfg *config.Control) error {
|
|
rand.Seed(time.Now().UTC().UnixNano())
|
|
|
|
runtime := &config.ControlRuntime{}
|
|
cfg.Runtime = runtime
|
|
|
|
if err := prepare(ctx, cfg, runtime); err != nil {
|
|
return errors.Wrap(err, "preparing server")
|
|
}
|
|
|
|
cfg.Runtime.Tunnel = setupTunnel()
|
|
util.DisableProxyHostnameCheck = true
|
|
|
|
var auth authenticator.Request
|
|
var handler http.Handler
|
|
var err error
|
|
|
|
if !cfg.DisableAPIServer {
|
|
auth, handler, err = apiServer(ctx, cfg, runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := waitForAPIServerInBackground(ctx, runtime); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
basicAuth, err := basicAuthenticator(runtime.PasswdFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
runtime.Authenticator = combineAuthenticators(basicAuth, auth)
|
|
runtime.Handler = handler
|
|
|
|
if !cfg.DisableScheduler {
|
|
if err := scheduler(cfg, runtime); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if !cfg.DisableControllerManager {
|
|
if err := controllerManager(cfg, runtime); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if !cfg.DisableCCM {
|
|
if err := cloudControllerManager(ctx, cfg, runtime); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) error {
|
|
argsMap := map[string]string{
|
|
"kubeconfig": runtime.KubeConfigController,
|
|
"service-account-private-key-file": runtime.ServiceKey,
|
|
"allocate-node-cidrs": "true",
|
|
"cluster-cidr": util2.JoinIPNets(cfg.ClusterIPRanges),
|
|
"root-ca-file": runtime.ServerCA,
|
|
"port": "10252",
|
|
"profiling": "false",
|
|
"address": localhostIP.String(),
|
|
"bind-address": localhostIP.String(),
|
|
"secure-port": "0",
|
|
"use-service-account-credentials": "true",
|
|
"cluster-signing-kube-apiserver-client-cert-file": runtime.ClientCA,
|
|
"cluster-signing-kube-apiserver-client-key-file": runtime.ClientCAKey,
|
|
"cluster-signing-kubelet-client-cert-file": runtime.ClientCA,
|
|
"cluster-signing-kubelet-client-key-file": runtime.ClientCAKey,
|
|
"cluster-signing-kubelet-serving-cert-file": runtime.ServerCA,
|
|
"cluster-signing-kubelet-serving-key-file": runtime.ServerCAKey,
|
|
"cluster-signing-legacy-unknown-cert-file": runtime.ClientCA,
|
|
"cluster-signing-legacy-unknown-key-file": runtime.ClientCAKey,
|
|
}
|
|
if cfg.NoLeaderElect {
|
|
argsMap["leader-elect"] = "false"
|
|
}
|
|
if !cfg.DisableCCM {
|
|
argsMap["configure-cloud-routes"] = "false"
|
|
argsMap["controllers"] = "*,-service,-route,-cloud-node-lifecycle"
|
|
}
|
|
|
|
args := config.GetArgs(argsMap, cfg.ExtraControllerArgs)
|
|
logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
|
|
|
|
return executor.ControllerManager(runtime.APIServerReady, args)
|
|
}
|
|
|
|
func scheduler(cfg *config.Control, runtime *config.ControlRuntime) error {
|
|
argsMap := map[string]string{
|
|
"kubeconfig": runtime.KubeConfigScheduler,
|
|
"port": "10251",
|
|
"address": "127.0.0.1",
|
|
"bind-address": "127.0.0.1",
|
|
"secure-port": "0",
|
|
"profiling": "false",
|
|
}
|
|
if cfg.NoLeaderElect {
|
|
argsMap["leader-elect"] = "false"
|
|
}
|
|
args := config.GetArgs(argsMap, cfg.ExtraSchedulerAPIArgs)
|
|
|
|
logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
|
|
return executor.Scheduler(runtime.APIServerReady, args)
|
|
}
|
|
|
|
func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
|
|
argsMap := make(map[string]string)
|
|
|
|
setupStorageBackend(argsMap, cfg)
|
|
|
|
certDir := filepath.Join(cfg.DataDir, "tls", "temporary-certs")
|
|
os.MkdirAll(certDir, 0700)
|
|
|
|
argsMap["cert-dir"] = certDir
|
|
argsMap["allow-privileged"] = "true"
|
|
argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")
|
|
argsMap["service-account-signing-key-file"] = runtime.ServiceKey
|
|
argsMap["service-cluster-ip-range"] = util2.JoinIPNets(cfg.ServiceIPRanges)
|
|
argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String()
|
|
argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)
|
|
if cfg.AdvertiseIP != "" {
|
|
argsMap["advertise-address"] = cfg.AdvertiseIP
|
|
}
|
|
argsMap["insecure-port"] = "0"
|
|
argsMap["secure-port"] = strconv.Itoa(cfg.APIServerPort)
|
|
if cfg.APIServerBindAddress == "" {
|
|
argsMap["bind-address"] = localhostIP.String()
|
|
} else {
|
|
argsMap["bind-address"] = cfg.APIServerBindAddress
|
|
}
|
|
argsMap["tls-cert-file"] = runtime.ServingKubeAPICert
|
|
argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey
|
|
argsMap["service-account-key-file"] = runtime.ServiceKey
|
|
argsMap["service-account-issuer"] = "https://kubernetes.default.svc." + cfg.ClusterDomain
|
|
argsMap["api-audiences"] = "https://kubernetes.default.svc." + cfg.ClusterDomain + "," + version.Program
|
|
argsMap["kubelet-certificate-authority"] = runtime.ServerCA
|
|
argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert
|
|
argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey
|
|
argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA
|
|
argsMap["requestheader-allowed-names"] = deps.RequestHeaderCN
|
|
argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert
|
|
argsMap["proxy-client-key-file"] = runtime.ClientAuthProxyKey
|
|
argsMap["requestheader-extra-headers-prefix"] = "X-Remote-Extra-"
|
|
argsMap["requestheader-group-headers"] = "X-Remote-Group"
|
|
argsMap["requestheader-username-headers"] = "X-Remote-User"
|
|
argsMap["client-ca-file"] = runtime.ClientCA
|
|
argsMap["enable-admission-plugins"] = "NodeRestriction"
|
|
argsMap["anonymous-auth"] = "false"
|
|
argsMap["profiling"] = "false"
|
|
if cfg.EncryptSecrets {
|
|
argsMap["encryption-provider-config"] = runtime.EncryptionConfig
|
|
}
|
|
args := config.GetArgs(argsMap, cfg.ExtraAPIArgs)
|
|
|
|
logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
|
|
|
|
return executor.APIServer(ctx, runtime.ETCDReady, args)
|
|
}
|
|
|
|
func defaults(config *config.Control) {
|
|
if config.ClusterIPRange == nil {
|
|
_, clusterIPNet, _ := net.ParseCIDR("10.42.0.0/16")
|
|
config.ClusterIPRange = clusterIPNet
|
|
}
|
|
|
|
if config.ServiceIPRange == nil {
|
|
_, serviceIPNet, _ := net.ParseCIDR("10.43.0.0/16")
|
|
config.ServiceIPRange = serviceIPNet
|
|
}
|
|
|
|
if len(config.ClusterDNS) == 0 {
|
|
config.ClusterDNS = net.ParseIP("10.43.0.10")
|
|
}
|
|
|
|
if config.AdvertisePort == 0 {
|
|
config.AdvertisePort = config.HTTPSPort
|
|
}
|
|
|
|
if config.APIServerPort == 0 {
|
|
if config.HTTPSPort != 0 {
|
|
config.APIServerPort = config.HTTPSPort + 1
|
|
} else {
|
|
config.APIServerPort = 6444
|
|
}
|
|
}
|
|
|
|
if config.DataDir == "" {
|
|
config.DataDir = "./management-state"
|
|
}
|
|
}
|
|
|
|
func prepare(ctx context.Context, config *config.Control, runtime *config.ControlRuntime) error {
|
|
var err error
|
|
|
|
defaults(config)
|
|
|
|
if err := os.MkdirAll(config.DataDir, 0700); err != nil {
|
|
return err
|
|
}
|
|
|
|
config.DataDir, err = filepath.Abs(config.DataDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
os.MkdirAll(filepath.Join(config.DataDir, "tls"), 0700)
|
|
os.MkdirAll(filepath.Join(config.DataDir, "cred"), 0700)
|
|
|
|
deps.FillRuntimeCerts(config, runtime)
|
|
|
|
cluster := cluster.New(config)
|
|
|
|
if err := cluster.Bootstrap(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := deps.GenServerDeps(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
ready, err := cluster.Start(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
runtime.ETCDReady = ready
|
|
return nil
|
|
}
|
|
|
|
func setupStorageBackend(argsMap map[string]string, cfg *config.Control) {
|
|
argsMap["storage-backend"] = "etcd3"
|
|
// specify the endpoints
|
|
if len(cfg.Datastore.Endpoint) > 0 {
|
|
argsMap["etcd-servers"] = cfg.Datastore.Endpoint
|
|
}
|
|
// storage backend tls configuration
|
|
if len(cfg.Datastore.CAFile) > 0 {
|
|
argsMap["etcd-cafile"] = cfg.Datastore.CAFile
|
|
}
|
|
if len(cfg.Datastore.CertFile) > 0 {
|
|
argsMap["etcd-certfile"] = cfg.Datastore.CertFile
|
|
}
|
|
if len(cfg.Datastore.KeyFile) > 0 {
|
|
argsMap["etcd-keyfile"] = cfg.Datastore.KeyFile
|
|
}
|
|
}
|
|
|
|
func cloudControllerManager(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) error {
|
|
argsMap := map[string]string{
|
|
"profiling": "false",
|
|
"allocate-node-cidrs": "true", // ccmOptions.KubeCloudShared.AllocateNodeCIDRs = true
|
|
"cloud-provider": version.Program, // ccmOptions.KubeCloudShared.CloudProvider.Name = version.Program
|
|
"cluster-cidr": util2.JoinIPNets(cfg.ClusterIPRanges), // ccmOptions.KubeCloudShared.ClusterCIDR = util2.JoinIPNets(cfg.ClusterIPRanges)
|
|
"configure-cloud-routes": "false", // ccmOptions.KubeCloudShared.ConfigureCloudRoutes = false
|
|
"kubeconfig": runtime.KubeConfigCloudController, // ccmOptions.Kubeconfig = runtime.KubeConfigCloudController
|
|
"node-status-update-frequency": "1m0s", // ccmOptions.NodeStatusUpdateFrequency = metav1.Duration{Duration: 1 * time.Minute}
|
|
"bind-address": "127.0.0.1", // ccmOptions.SecureServing.BindAddress = localhostIP
|
|
"port": "0", // ccmOptions.SecureServing.BindPort = 0
|
|
}
|
|
if cfg.NoLeaderElect {
|
|
argsMap["leader-elect"] = "false"
|
|
}
|
|
args := config.GetArgs(argsMap, cfg.ExtraCloudControllerArgs)
|
|
|
|
logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args))
|
|
|
|
ccmRBACReady := make(chan struct{})
|
|
|
|
go func() {
|
|
defer close(ccmRBACReady)
|
|
|
|
apiReadyLoop:
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-runtime.APIServerReady:
|
|
break apiReadyLoop
|
|
case <-time.After(30 * time.Second):
|
|
logrus.Infof("Waiting for API server to become available")
|
|
}
|
|
}
|
|
|
|
logrus.Infof("Waiting for cloud-controller-manager privileges to become available")
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case err := <-promise(func() error { return checkForCloudControllerPrivileges(runtime, 5*time.Second) }):
|
|
if err != nil {
|
|
logrus.Infof("Waiting for cloud-controller-manager privileges to become available")
|
|
continue
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
return executor.CloudControllerManager(ccmRBACReady, args)
|
|
}
|
|
|
|
func checkForCloudControllerPrivileges(runtime *config.ControlRuntime, timeout time.Duration) error {
|
|
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = wait.PollImmediate(time.Second, timeout, func() (bool, error) {
|
|
crb := rbac.NewFactoryFromConfigOrDie(restConfig).Rbac().V1().ClusterRoleBinding()
|
|
_, err = crb.Get(version.Program+"-cloud-controller-manager", metav1.GetOptions{})
|
|
if err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
})
|
|
|
|
if err != nil {
|
|
logrus.Errorf("error encountered waitng for cloud-controller-manager privileges: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRuntime) error {
|
|
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
k8sClient, err := kubernetes.NewForConfig(restConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
done := make(chan struct{})
|
|
runtime.APIServerReady = done
|
|
|
|
go func() {
|
|
defer close(done)
|
|
|
|
etcdLoop:
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-runtime.ETCDReady:
|
|
break etcdLoop
|
|
case <-time.After(30 * time.Second):
|
|
logrus.Infof("Waiting for etcd server to become available")
|
|
}
|
|
}
|
|
|
|
logrus.Infof("Waiting for API server to become available")
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 30*time.Second) }):
|
|
if err != nil {
|
|
logrus.Infof("Waiting for API server to become available")
|
|
continue
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
return nil
|
|
}
|
|
|
|
func promise(f func() error) <-chan error {
|
|
c := make(chan error, 1)
|
|
go func() {
|
|
c <- f()
|
|
close(c)
|
|
}()
|
|
return c
|
|
}
|