mirror of
https://github.com/k3s-io/k3s.git
synced 2024-06-07 19:41:36 +00:00
6b5b69378f
This is replaces dqlite with etcd. The each same UX of dqlite is followed so there is no change to the CLI args for this.
1055 lines
30 KiB
Go
1055 lines
30 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"crypto"
|
|
cryptorand "crypto/rand"
|
|
"crypto/x509"
|
|
b64 "encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"text/template"
|
|
"time"
|
|
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
|
|
"github.com/pkg/errors"
|
|
certutil "github.com/rancher/dynamiclistener/cert"
|
|
"github.com/rancher/k3s/pkg/clientaccess"
|
|
"github.com/rancher/k3s/pkg/cluster"
|
|
"github.com/rancher/k3s/pkg/daemons/config"
|
|
"github.com/rancher/k3s/pkg/daemons/executor"
|
|
"github.com/rancher/k3s/pkg/passwd"
|
|
"github.com/rancher/k3s/pkg/token"
|
|
"github.com/rancher/k3s/pkg/version"
|
|
"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
|
|
"github.com/sirupsen/logrus"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
|
|
"k8s.io/apiserver/pkg/authentication/authenticator"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/tools/clientcmd"
|
|
ccmapp "k8s.io/kubernetes/cmd/cloud-controller-manager/app"
|
|
app2 "k8s.io/kubernetes/cmd/controller-manager/app"
|
|
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
|
|
"k8s.io/kubernetes/pkg/master"
|
|
"k8s.io/kubernetes/pkg/proxy/util"
|
|
|
|
// registering k3s cloud provider
|
|
_ "github.com/rancher/k3s/pkg/cloudprovider"
|
|
// for client metric registration
|
|
_ "k8s.io/component-base/metrics/prometheus/restclient"
|
|
)
|
|
|
|
var (
|
|
localhostIP = net.ParseIP("127.0.0.1")
|
|
requestHeaderCN = "system:auth-proxy"
|
|
kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
|
|
clusters:
|
|
- cluster:
|
|
server: {{.URL}}
|
|
certificate-authority: {{.CACert}}
|
|
name: local
|
|
contexts:
|
|
- context:
|
|
cluster: local
|
|
namespace: default
|
|
user: user
|
|
name: Default
|
|
current-context: Default
|
|
kind: Config
|
|
preferences: {}
|
|
users:
|
|
- name: user
|
|
user:
|
|
client-certificate: {{.ClientCert}}
|
|
client-key: {{.ClientKey}}
|
|
`))
|
|
)
|
|
|
|
const (
|
|
ipsecTokenSize = 48
|
|
aescbcKeySize = 32
|
|
)
|
|
|
|
func Server(ctx context.Context, cfg *config.Control) error {
|
|
rand.Seed(time.Now().UTC().UnixNano())
|
|
|
|
runtime := &config.ControlRuntime{}
|
|
cfg.Runtime = runtime
|
|
|
|
if err := prepare(ctx, cfg, runtime); err != nil {
|
|
return errors.Wrap(err, "preparing server")
|
|
}
|
|
|
|
cfg.Runtime.Tunnel = setupTunnel()
|
|
util.DisableProxyHostnameCheck = true
|
|
|
|
auth, handler, err := apiServer(ctx, cfg, runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := waitForAPIServerInBackground(ctx, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
runtime.Handler = handler
|
|
runtime.Authenticator = auth
|
|
|
|
if !cfg.NoScheduler {
|
|
if err := scheduler(cfg, runtime); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := controllerManager(cfg, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if !cfg.DisableCCM {
|
|
cloudControllerManager(ctx, cfg, runtime)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) error {
|
|
argsMap := map[string]string{
|
|
"kubeconfig": runtime.KubeConfigController,
|
|
"service-account-private-key-file": runtime.ServiceKey,
|
|
"allocate-node-cidrs": "true",
|
|
"cluster-cidr": cfg.ClusterIPRange.String(),
|
|
"root-ca-file": runtime.ServerCA,
|
|
"port": "10252",
|
|
"bind-address": localhostIP.String(),
|
|
"secure-port": "0",
|
|
"use-service-account-credentials": "true",
|
|
"cluster-signing-cert-file": runtime.ServerCA,
|
|
"cluster-signing-key-file": runtime.ServerCAKey,
|
|
}
|
|
if cfg.NoLeaderElect {
|
|
argsMap["leader-elect"] = "false"
|
|
}
|
|
|
|
args := config.GetArgsList(argsMap, cfg.ExtraControllerArgs)
|
|
logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
|
|
|
|
return executor.ControllerManager(runtime.APIServerReady, args)
|
|
}
|
|
|
|
func scheduler(cfg *config.Control, runtime *config.ControlRuntime) error {
|
|
argsMap := map[string]string{
|
|
"kubeconfig": runtime.KubeConfigScheduler,
|
|
"port": "10251",
|
|
"bind-address": "127.0.0.1",
|
|
"secure-port": "0",
|
|
}
|
|
if cfg.NoLeaderElect {
|
|
argsMap["leader-elect"] = "false"
|
|
}
|
|
args := config.GetArgsList(argsMap, cfg.ExtraSchedulerAPIArgs)
|
|
|
|
logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
|
|
return executor.Scheduler(runtime.APIServerReady, args)
|
|
}
|
|
|
|
func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
|
|
argsMap := make(map[string]string)
|
|
|
|
setupStorageBackend(argsMap, cfg)
|
|
|
|
certDir := filepath.Join(cfg.DataDir, "tls", "temporary-certs")
|
|
os.MkdirAll(certDir, 0700)
|
|
|
|
argsMap["cert-dir"] = certDir
|
|
argsMap["allow-privileged"] = "true"
|
|
argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")
|
|
argsMap["service-account-signing-key-file"] = runtime.ServiceKey
|
|
argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String()
|
|
argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)
|
|
if cfg.AdvertiseIP != "" {
|
|
argsMap["advertise-address"] = cfg.AdvertiseIP
|
|
}
|
|
argsMap["insecure-port"] = "0"
|
|
argsMap["secure-port"] = strconv.Itoa(cfg.APIServerPort)
|
|
if cfg.APIServerBindAddress == "" {
|
|
argsMap["bind-address"] = localhostIP.String()
|
|
} else {
|
|
argsMap["bind-address"] = cfg.APIServerBindAddress
|
|
}
|
|
argsMap["tls-cert-file"] = runtime.ServingKubeAPICert
|
|
argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey
|
|
argsMap["service-account-key-file"] = runtime.ServiceKey
|
|
argsMap["service-account-issuer"] = version.Program
|
|
argsMap["api-audiences"] = "unknown"
|
|
argsMap["basic-auth-file"] = runtime.PasswdFile
|
|
argsMap["kubelet-certificate-authority"] = runtime.ServerCA
|
|
argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert
|
|
argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey
|
|
argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA
|
|
argsMap["requestheader-allowed-names"] = requestHeaderCN
|
|
argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert
|
|
argsMap["proxy-client-key-file"] = runtime.ClientAuthProxyKey
|
|
argsMap["requestheader-extra-headers-prefix"] = "X-Remote-Extra-"
|
|
argsMap["requestheader-group-headers"] = "X-Remote-Group"
|
|
argsMap["requestheader-username-headers"] = "X-Remote-User"
|
|
argsMap["client-ca-file"] = runtime.ClientCA
|
|
argsMap["enable-admission-plugins"] = "NodeRestriction"
|
|
argsMap["anonymous-auth"] = "false"
|
|
if cfg.EncryptSecrets {
|
|
argsMap["encryption-provider-config"] = runtime.EncryptionConfig
|
|
}
|
|
args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs)
|
|
|
|
logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
|
|
return executor.APIServer(ctx, runtime.ETCDReady, args)
|
|
}
|
|
|
|
func defaults(config *config.Control) {
|
|
if config.ClusterIPRange == nil {
|
|
_, clusterIPNet, _ := net.ParseCIDR("10.42.0.0/16")
|
|
config.ClusterIPRange = clusterIPNet
|
|
}
|
|
|
|
if config.ServiceIPRange == nil {
|
|
_, serviceIPNet, _ := net.ParseCIDR("10.43.0.0/16")
|
|
config.ServiceIPRange = serviceIPNet
|
|
}
|
|
|
|
if len(config.ClusterDNS) == 0 {
|
|
config.ClusterDNS = net.ParseIP("10.43.0.10")
|
|
}
|
|
|
|
if config.AdvertisePort == 0 {
|
|
config.AdvertisePort = config.HTTPSPort
|
|
}
|
|
|
|
if config.APIServerPort == 0 {
|
|
if config.HTTPSPort != 0 {
|
|
config.APIServerPort = config.HTTPSPort + 1
|
|
} else {
|
|
config.APIServerPort = 6444
|
|
}
|
|
}
|
|
|
|
if config.DataDir == "" {
|
|
config.DataDir = "./management-state"
|
|
}
|
|
}
|
|
|
|
func prepare(ctx context.Context, config *config.Control, runtime *config.ControlRuntime) error {
|
|
var err error
|
|
|
|
defaults(config)
|
|
|
|
if err := os.MkdirAll(config.DataDir, 0700); err != nil {
|
|
return err
|
|
}
|
|
|
|
config.DataDir, err = filepath.Abs(config.DataDir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
os.MkdirAll(filepath.Join(config.DataDir, "tls"), 0700)
|
|
os.MkdirAll(filepath.Join(config.DataDir, "cred"), 0700)
|
|
|
|
runtime.ClientCA = filepath.Join(config.DataDir, "tls", "client-ca.crt")
|
|
runtime.ClientCAKey = filepath.Join(config.DataDir, "tls", "client-ca.key")
|
|
runtime.ServerCA = filepath.Join(config.DataDir, "tls", "server-ca.crt")
|
|
runtime.ServerCAKey = filepath.Join(config.DataDir, "tls", "server-ca.key")
|
|
runtime.RequestHeaderCA = filepath.Join(config.DataDir, "tls", "request-header-ca.crt")
|
|
runtime.RequestHeaderCAKey = filepath.Join(config.DataDir, "tls", "request-header-ca.key")
|
|
runtime.IPSECKey = filepath.Join(config.DataDir, "cred", "ipsec.psk")
|
|
|
|
runtime.ServiceKey = filepath.Join(config.DataDir, "tls", "service.key")
|
|
runtime.PasswdFile = filepath.Join(config.DataDir, "cred", "passwd")
|
|
runtime.NodePasswdFile = filepath.Join(config.DataDir, "cred", "node-passwd")
|
|
|
|
runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig")
|
|
runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig")
|
|
runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig")
|
|
runtime.KubeConfigAPIServer = filepath.Join(config.DataDir, "cred", "api-server.kubeconfig")
|
|
runtime.KubeConfigCloudController = filepath.Join(config.DataDir, "cred", "cloud-controller.kubeconfig")
|
|
|
|
runtime.ClientAdminCert = filepath.Join(config.DataDir, "tls", "client-admin.crt")
|
|
runtime.ClientAdminKey = filepath.Join(config.DataDir, "tls", "client-admin.key")
|
|
runtime.ClientControllerCert = filepath.Join(config.DataDir, "tls", "client-controller.crt")
|
|
runtime.ClientControllerKey = filepath.Join(config.DataDir, "tls", "client-controller.key")
|
|
runtime.ClientCloudControllerCert = filepath.Join(config.DataDir, "tls", "client-cloud-controller.crt")
|
|
runtime.ClientCloudControllerKey = filepath.Join(config.DataDir, "tls", "client-cloud-controller.key")
|
|
runtime.ClientSchedulerCert = filepath.Join(config.DataDir, "tls", "client-scheduler.crt")
|
|
runtime.ClientSchedulerKey = filepath.Join(config.DataDir, "tls", "client-scheduler.key")
|
|
runtime.ClientKubeAPICert = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.crt")
|
|
runtime.ClientKubeAPIKey = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.key")
|
|
runtime.ClientKubeProxyCert = filepath.Join(config.DataDir, "tls", "client-kube-proxy.crt")
|
|
runtime.ClientKubeProxyKey = filepath.Join(config.DataDir, "tls", "client-kube-proxy.key")
|
|
runtime.ClientK3sControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.crt")
|
|
runtime.ClientK3sControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.key")
|
|
|
|
runtime.ServingKubeAPICert = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
|
|
runtime.ServingKubeAPIKey = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
|
|
|
|
runtime.ClientKubeletKey = filepath.Join(config.DataDir, "tls", "client-kubelet.key")
|
|
runtime.ServingKubeletKey = filepath.Join(config.DataDir, "tls", "serving-kubelet.key")
|
|
|
|
runtime.ClientAuthProxyCert = filepath.Join(config.DataDir, "tls", "client-auth-proxy.crt")
|
|
runtime.ClientAuthProxyKey = filepath.Join(config.DataDir, "tls", "client-auth-proxy.key")
|
|
|
|
runtime.ETCDServerCA = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.crt")
|
|
runtime.ETCDServerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.key")
|
|
runtime.ETCDPeerCA = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.crt")
|
|
runtime.ETCDPeerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.key")
|
|
runtime.ServerETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "server-client.crt")
|
|
runtime.ServerETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "server-client.key")
|
|
runtime.PeerServerClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.crt")
|
|
runtime.PeerServerClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.key")
|
|
runtime.ClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "client.crt")
|
|
runtime.ClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "client.key")
|
|
|
|
if config.EncryptSecrets {
|
|
runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json")
|
|
}
|
|
|
|
cluster := cluster.New(config)
|
|
|
|
if err := cluster.Bootstrap(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genServiceAccount(runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genUsers(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genEncryptedNetworkInfo(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := genEncryptionConfig(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := readTokens(runtime); err != nil {
|
|
return err
|
|
}
|
|
|
|
ready, err := cluster.Start(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
runtime.ETCDReady = ready
|
|
return nil
|
|
}
|
|
|
|
func readTokens(runtime *config.ControlRuntime) error {
|
|
tokens, err := passwd.Read(runtime.PasswdFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if nodeToken, ok := tokens.Pass("node"); ok {
|
|
runtime.AgentToken = "node:" + nodeToken
|
|
}
|
|
if serverToken, ok := tokens.Pass("server"); ok {
|
|
runtime.ServerToken = "server:" + serverToken
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
|
|
if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
|
|
psk, err := ioutil.ReadFile(runtime.IPSECKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
controlConfig.IPSECPSK = strings.TrimSpace(string(psk))
|
|
return nil
|
|
}
|
|
|
|
psk, err := token.Random(ipsecTokenSize)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
controlConfig.IPSECPSK = psk
|
|
if err := ioutil.WriteFile(runtime.IPSECKey, []byte(psk+"\n"), 0600); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func migratePassword(p *passwd.Passwd) error {
|
|
server, _ := p.Pass("server")
|
|
node, _ := p.Pass("node")
|
|
if server == "" && node != "" {
|
|
return p.EnsureUser("server", version.Program+":server", node)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) {
|
|
var (
|
|
err error
|
|
)
|
|
|
|
serverPass := config.Token
|
|
if serverPass == "" {
|
|
serverPass, _ = passwd.Pass("server")
|
|
}
|
|
if serverPass == "" {
|
|
serverPass, err = token.Random(16)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
|
|
return serverPass, nil
|
|
}
|
|
|
|
func getNodePass(config *config.Control, serverPass string) string {
|
|
if config.AgentToken == "" {
|
|
if _, passwd, ok := clientaccess.ParseUsernamePassword(serverPass); ok {
|
|
return passwd
|
|
}
|
|
return serverPass
|
|
}
|
|
return config.AgentToken
|
|
}
|
|
|
|
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
|
|
passwd, err := passwd.Read(runtime.PasswdFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := migratePassword(passwd); err != nil {
|
|
return err
|
|
}
|
|
|
|
serverPass, err := getServerPass(passwd, config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
nodePass := getNodePass(config, serverPass)
|
|
|
|
if err := passwd.EnsureUser("node", version.Program+":agent", nodePass); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := passwd.EnsureUser("server", version.Program+":server", serverPass); err != nil {
|
|
return err
|
|
}
|
|
|
|
return passwd.Write(runtime.PasswdFile)
|
|
}
|
|
|
|
func genCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
if err := genClientCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := genServerCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := genRequestHeaderCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
if err := genETCDCerts(config, runtime); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type signedCertFactory = func(commonName string, organization []string, certFile, keyFile string) (bool, error)
|
|
|
|
func getSigningCertFactory(regen bool, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile string) signedCertFactory {
|
|
return func(commonName string, organization []string, certFile, keyFile string) (bool, error) {
|
|
return createClientCertKey(regen, commonName, organization, altNames, extKeyUsage, caCertFile, caKeyFile, certFile, keyFile)
|
|
}
|
|
}
|
|
|
|
func genClientCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
regen, err := createSigningCertKey(version.Program+"-client", runtime.ClientCA, runtime.ClientCAKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
factory := getSigningCertFactory(regen, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.ClientCA, runtime.ClientCAKey)
|
|
|
|
var certGen bool
|
|
apiEndpoint := fmt.Sprintf("https://127.0.0.1:%d", config.APIServerPort)
|
|
|
|
certGen, err = factory("system:admin", []string{"system:masters"}, runtime.ClientAdminCert, runtime.ClientAdminKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigAdmin, apiEndpoint, runtime.ServerCA, runtime.ClientAdminCert, runtime.ClientAdminKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
certGen, err = factory("system:kube-controller-manager", nil, runtime.ClientControllerCert, runtime.ClientControllerKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigController, apiEndpoint, runtime.ServerCA, runtime.ClientControllerCert, runtime.ClientControllerKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
certGen, err = factory("system:kube-scheduler", nil, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigScheduler, apiEndpoint, runtime.ServerCA, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
certGen, err = factory("kube-apiserver", nil, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigAPIServer, apiEndpoint, runtime.ServerCA, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if _, err = factory("system:kube-proxy", nil, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
|
|
return err
|
|
}
|
|
// this must be hardcoded to k3s-controller because it's hard coded in the rolebindings.yaml
|
|
if _, err = factory("system:k3s-controller", nil, runtime.ClientK3sControllerCert, runtime.ClientK3sControllerKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ClientKubeletKey, regen); err != nil {
|
|
return err
|
|
}
|
|
|
|
certGen, err = factory("cloud-controller-manager", nil, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if certGen {
|
|
if err := KubeConfig(runtime.KubeConfigCloudController, apiEndpoint, runtime.ServerCA, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func createServerSigningCertKey(config *config.Control, runtime *config.ControlRuntime) (bool, error) {
|
|
TokenCA := filepath.Join(config.DataDir, "tls", "token-ca.crt")
|
|
TokenCAKey := filepath.Join(config.DataDir, "tls", "token-ca.key")
|
|
|
|
if exists(TokenCA, TokenCAKey) && !exists(runtime.ServerCA) && !exists(runtime.ServerCAKey) {
|
|
logrus.Infof("Upgrading token-ca files to server-ca")
|
|
if err := os.Link(TokenCA, runtime.ServerCA); err != nil {
|
|
return false, err
|
|
}
|
|
if err := os.Link(TokenCAKey, runtime.ServerCAKey); err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
return createSigningCertKey(version.Program+"-server", runtime.ServerCA, runtime.ServerCAKey)
|
|
}
|
|
|
|
func addSANs(altNames *certutil.AltNames, sans []string) {
|
|
for _, san := range sans {
|
|
ip := net.ParseIP(san)
|
|
if ip == nil {
|
|
altNames.DNSNames = append(altNames.DNSNames, san)
|
|
} else {
|
|
altNames.IPs = append(altNames.IPs, ip)
|
|
}
|
|
}
|
|
}
|
|
|
|
func genServerCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
regen, err := createServerSigningCertKey(config, runtime)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, apiServerServiceIP, err := master.ServiceIPRange(*config.ServiceIPRange)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
altNames := &certutil.AltNames{
|
|
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
|
|
IPs: []net.IP{apiServerServiceIP},
|
|
}
|
|
|
|
addSANs(altNames, config.SANs)
|
|
|
|
if _, err := createClientCertKey(regen, "kube-apiserver", nil,
|
|
altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
|
runtime.ServerCA, runtime.ServerCAKey,
|
|
runtime.ServingKubeAPICert, runtime.ServingKubeAPIKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ServingKubeletKey, regen); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func genETCDCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
regen, err := createSigningCertKey("etcd-server", runtime.ETCDServerCA, runtime.ETCDServerCAKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
altNames := &certutil.AltNames{
|
|
DNSNames: []string{"localhost"},
|
|
}
|
|
addSANs(altNames, config.SANs)
|
|
|
|
if _, err := createClientCertKey(regen, "etcd-server", nil,
|
|
altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
|
|
runtime.ETCDServerCA, runtime.ETCDServerCAKey,
|
|
runtime.ServerETCDCert, runtime.ServerETCDKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := createClientCertKey(regen, "etcd-client", nil,
|
|
nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
|
runtime.ETCDServerCA, runtime.ETCDServerCAKey,
|
|
runtime.ClientETCDCert, runtime.ClientETCDKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
regen, err = createSigningCertKey("etcd-peer", runtime.ETCDPeerCA, runtime.ETCDPeerCAKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := createClientCertKey(regen, "etcd-peer", nil,
|
|
altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
|
|
runtime.ETCDPeerCA, runtime.ETCDPeerCAKey,
|
|
runtime.PeerServerClientETCDCert, runtime.PeerServerClientETCDKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntime) error {
|
|
regen, err := createSigningCertKey(version.Program+"-request-header", runtime.RequestHeaderCA, runtime.RequestHeaderCAKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if _, err := createClientCertKey(regen, requestHeaderCN, nil,
|
|
nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
|
|
runtime.RequestHeaderCA, runtime.RequestHeaderCAKey,
|
|
runtime.ClientAuthProxyCert, runtime.ClientAuthProxyKey); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) (bool, error) {
|
|
caBytes, err := ioutil.ReadFile(caCertFile)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
pool := x509.NewCertPool()
|
|
pool.AppendCertsFromPEM(caBytes)
|
|
|
|
// check for certificate expiration
|
|
if !regen {
|
|
regen = expired(certFile, pool)
|
|
}
|
|
|
|
if !regen {
|
|
regen = sansChanged(certFile, altNames)
|
|
}
|
|
|
|
if !regen {
|
|
if exists(certFile, keyFile) {
|
|
return false, nil
|
|
}
|
|
}
|
|
|
|
caKeyBytes, err := ioutil.ReadFile(caKeyFile)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
caCert, err := certutil.ParseCertsPEM(caBytes)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, regen)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
cfg := certutil.Config{
|
|
CommonName: commonName,
|
|
Organization: organization,
|
|
Usages: extKeyUsage,
|
|
}
|
|
if altNames != nil {
|
|
cfg.AltNames = *altNames
|
|
}
|
|
cert, err := certutil.NewSignedCert(cfg, key.(crypto.Signer), caCert[0], caKey.(crypto.Signer))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
return true, certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
|
|
}
|
|
|
|
func exists(files ...string) bool {
|
|
for _, file := range files {
|
|
if _, err := os.Stat(file); err != nil {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func genServiceAccount(runtime *config.ControlRuntime) error {
|
|
_, keyErr := os.Stat(runtime.ServiceKey)
|
|
if keyErr == nil {
|
|
return nil
|
|
}
|
|
|
|
key, err := certutil.NewPrivateKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key))
|
|
}
|
|
|
|
func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) {
|
|
if exists(certFile, keyFile) {
|
|
return false, nil
|
|
}
|
|
|
|
caKeyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
cfg := certutil.Config{
|
|
CommonName: fmt.Sprintf("%s-ca@%d", prefix, time.Now().Unix()),
|
|
}
|
|
|
|
cert, err := certutil.NewSelfSignedCACert(cfg, caKey.(crypto.Signer))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if err := certutil.WriteCert(certFile, certutil.EncodeCertPEM(cert)); err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func KubeConfig(dest, url, caCert, clientCert, clientKey string) error {
|
|
data := struct {
|
|
URL string
|
|
CACert string
|
|
ClientCert string
|
|
ClientKey string
|
|
}{
|
|
URL: url,
|
|
CACert: caCert,
|
|
ClientCert: clientCert,
|
|
ClientKey: clientKey,
|
|
}
|
|
|
|
output, err := os.Create(dest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer output.Close()
|
|
|
|
return kubeconfigTemplate.Execute(output, &data)
|
|
}
|
|
|
|
func setupStorageBackend(argsMap map[string]string, cfg *config.Control) {
|
|
argsMap["storage-backend"] = "etcd3"
|
|
// specify the endpoints
|
|
if len(cfg.Datastore.Endpoint) > 0 {
|
|
argsMap["etcd-servers"] = cfg.Datastore.Endpoint
|
|
}
|
|
// storage backend tls configuration
|
|
if len(cfg.Datastore.CAFile) > 0 {
|
|
argsMap["etcd-cafile"] = cfg.Datastore.CAFile
|
|
}
|
|
if len(cfg.Datastore.CertFile) > 0 {
|
|
argsMap["etcd-certfile"] = cfg.Datastore.CertFile
|
|
}
|
|
if len(cfg.Datastore.KeyFile) > 0 {
|
|
argsMap["etcd-keyfile"] = cfg.Datastore.KeyFile
|
|
}
|
|
}
|
|
|
|
func sansChanged(certFile string, sans *certutil.AltNames) bool {
|
|
if sans == nil {
|
|
return false
|
|
}
|
|
|
|
certBytes, err := ioutil.ReadFile(certFile)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
certificates, err := certutil.ParseCertsPEM(certBytes)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
if len(certificates) == 0 {
|
|
return false
|
|
}
|
|
|
|
if !sets.NewString(certificates[0].DNSNames...).HasAll(sans.DNSNames...) {
|
|
return true
|
|
}
|
|
|
|
ips := sets.NewString()
|
|
for _, ip := range certificates[0].IPAddresses {
|
|
ips.Insert(ip.String())
|
|
}
|
|
|
|
for _, ip := range sans.IPs {
|
|
if !ips.Has(ip.String()) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func expired(certFile string, pool *x509.CertPool) bool {
|
|
certBytes, err := ioutil.ReadFile(certFile)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
certificates, err := certutil.ParseCertsPEM(certBytes)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
_, err = certificates[0].Verify(x509.VerifyOptions{
|
|
Roots: pool,
|
|
KeyUsages: []x509.ExtKeyUsage{
|
|
x509.ExtKeyUsageAny,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return true
|
|
}
|
|
return certutil.IsCertExpired(certificates[0])
|
|
}
|
|
|
|
func cloudControllerManager(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) {
|
|
argsMap := map[string]string{
|
|
"kubeconfig": runtime.KubeConfigCloudController,
|
|
"allocate-node-cidrs": "true",
|
|
"cluster-cidr": cfg.ClusterIPRange.String(),
|
|
"bind-address": localhostIP.String(),
|
|
"secure-port": "0",
|
|
"cloud-provider": version.Program,
|
|
"allow-untagged-cloud": "true",
|
|
"node-status-update-frequency": "1m",
|
|
}
|
|
if cfg.NoLeaderElect {
|
|
argsMap["leader-elect"] = "false"
|
|
}
|
|
|
|
args := config.GetArgsList(argsMap, cfg.ExtraCloudControllerArgs)
|
|
|
|
command := ccmapp.NewCloudControllerManagerCommand()
|
|
command.SetArgs(args)
|
|
// register k3s cloud provider
|
|
|
|
go func() {
|
|
for {
|
|
// check for the cloud controller rbac binding
|
|
if err := checkForCloudControllerPrivileges(runtime); err != nil {
|
|
logrus.Infof("Waiting for cloudcontroller rbac role to be created")
|
|
select {
|
|
case <-ctx.Done():
|
|
logrus.Fatalf("cloud-controller-manager context canceled: %v", ctx.Err())
|
|
case <-time.After(time.Second):
|
|
continue
|
|
}
|
|
}
|
|
break
|
|
}
|
|
logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args))
|
|
logrus.Fatalf("cloud-controller-manager exited: %v", command.Execute())
|
|
}()
|
|
}
|
|
|
|
func checkForCloudControllerPrivileges(runtime *config.ControlRuntime) error {
|
|
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
crb := rbac.NewFactoryFromConfigOrDie(restConfig).Rbac().V1().ClusterRoleBinding()
|
|
_, err = crb.Get("cloud-controller-manager", metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRuntime) error {
|
|
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
k8sClient, err := kubernetes.NewForConfig(restConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
done := make(chan struct{})
|
|
runtime.APIServerReady = done
|
|
|
|
go func() {
|
|
defer close(done)
|
|
|
|
etcdLoop:
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-runtime.ETCDReady:
|
|
break etcdLoop
|
|
case <-time.After(30 * time.Second):
|
|
logrus.Infof("Waiting for etcd server to become available")
|
|
}
|
|
}
|
|
|
|
logrus.Infof("Waiting for API server to become available")
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 30*time.Second) }):
|
|
if err != nil {
|
|
logrus.Infof("Waiting for API server to become available")
|
|
continue
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
return nil
|
|
}
|
|
|
|
func promise(f func() error) <-chan error {
|
|
c := make(chan error, 1)
|
|
go func() {
|
|
c <- f()
|
|
close(c)
|
|
}()
|
|
return c
|
|
}
|
|
|
|
func genEncryptionConfig(controlConfig *config.Control, runtime *config.ControlRuntime) error {
|
|
if !controlConfig.EncryptSecrets {
|
|
return nil
|
|
}
|
|
if s, err := os.Stat(runtime.EncryptionConfig); err == nil && s.Size() > 0 {
|
|
return nil
|
|
}
|
|
|
|
aescbcKey := make([]byte, aescbcKeySize, aescbcKeySize)
|
|
_, err := cryptorand.Read(aescbcKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
encodedKey := b64.StdEncoding.EncodeToString(aescbcKey)
|
|
|
|
encConfig := apiserverconfigv1.EncryptionConfiguration{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "EncryptionConfiguration",
|
|
APIVersion: "apiserver.config.k8s.io/v1",
|
|
},
|
|
Resources: []apiserverconfigv1.ResourceConfiguration{
|
|
{
|
|
Resources: []string{"secrets"},
|
|
Providers: []apiserverconfigv1.ProviderConfiguration{
|
|
{
|
|
AESCBC: &apiserverconfigv1.AESConfiguration{
|
|
Keys: []apiserverconfigv1.Key{
|
|
{
|
|
Name: "aescbckey",
|
|
Secret: encodedKey,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Identity: &apiserverconfigv1.IdentityConfiguration{},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
jsonfile, err := json.Marshal(encConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return ioutil.WriteFile(runtime.EncryptionConfig, jsonfile, 0600)
|
|
}
|