k3s/pkg/daemons/control/server.go

864 lines
25 KiB
Go
Raw Normal View History

2019-01-01 08:23:01 +00:00
package control
import (
"context"
2019-05-29 18:53:51 +00:00
"crypto"
2019-01-01 08:23:01 +00:00
"crypto/x509"
"fmt"
"html/template"
"io/ioutil"
"math/rand"
"net"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
2019-10-15 21:17:26 +00:00
// registering k3s cloud provider
_ "github.com/rancher/k3s/pkg/cloudprovider"
certutil "github.com/rancher/dynamiclistener/cert"
2019-11-08 20:58:50 +00:00
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/cluster"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/passwd"
"github.com/rancher/k3s/pkg/token"
2019-10-15 21:17:26 +00:00
"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
2019-01-01 08:23:01 +00:00
"github.com/sirupsen/logrus"
2019-10-15 21:17:26 +00:00
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2019-01-01 08:23:01 +00:00
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/client-go/kubernetes"
2019-10-15 21:17:26 +00:00
"k8s.io/client-go/tools/clientcmd"
ccmapp "k8s.io/kubernetes/cmd/cloud-controller-manager/app"
app2 "k8s.io/kubernetes/cmd/controller-manager/app"
2019-01-01 08:23:01 +00:00
"k8s.io/kubernetes/cmd/kube-apiserver/app"
cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
sapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
"k8s.io/kubernetes/pkg/master"
2019-02-08 04:14:02 +00:00
"k8s.io/kubernetes/pkg/proxy/util"
2019-01-01 08:23:01 +00:00
)
var (
localhostIP = net.ParseIP("127.0.0.1")
2019-05-29 18:53:51 +00:00
requestHeaderCN = "system:auth-proxy"
2019-01-01 08:23:01 +00:00
kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
clusters:
- cluster:
server: {{.URL}}
2019-05-29 18:53:51 +00:00
certificate-authority: {{.CACert}}
2019-01-01 08:23:01 +00:00
name: local
contexts:
- context:
cluster: local
namespace: default
user: user
name: Default
current-context: Default
kind: Config
preferences: {}
users:
- name: user
user:
2019-05-29 18:53:51 +00:00
client-certificate: {{.ClientCert}}
client-key: {{.ClientKey}}
2019-01-01 08:23:01 +00:00
`))
)
2019-09-03 23:41:54 +00:00
const (
userTokenSize = 16
ipsecTokenSize = 48
)
2019-01-01 08:23:01 +00:00
func Server(ctx context.Context, cfg *config.Control) error {
rand.Seed(time.Now().UTC().UnixNano())
runtime := &config.ControlRuntime{}
cfg.Runtime = runtime
if err := prepare(ctx, cfg, runtime); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
cfg.Runtime.Tunnel = setupTunnel()
2019-02-08 04:14:02 +00:00
util.DisableProxyHostnameCheck = true
2019-01-01 08:23:01 +00:00
auth, handler, err := apiServer(ctx, cfg, runtime)
if err != nil {
return err
}
if err := waitForAPIServer(ctx, runtime); err != nil {
return err
}
2019-01-01 08:23:01 +00:00
runtime.Handler = handler
runtime.Authenticator = auth
if !cfg.NoScheduler {
scheduler(cfg, runtime)
}
controllerManager(cfg, runtime)
2019-10-15 21:17:26 +00:00
if !cfg.DisableCCM {
cloudControllerManager(ctx, cfg, runtime)
2019-10-15 21:17:26 +00:00
}
2019-01-01 08:23:01 +00:00
return nil
}
2019-01-09 16:54:15 +00:00
func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
argsMap := map[string]string{
2019-05-29 18:53:51 +00:00
"kubeconfig": runtime.KubeConfigController,
"service-account-private-key-file": runtime.ServiceKey,
"allocate-node-cidrs": "true",
"cluster-cidr": cfg.ClusterIPRange.String(),
2019-05-29 18:53:51 +00:00
"root-ca-file": runtime.ServerCA,
"port": "10252",
2019-05-29 18:53:51 +00:00
"bind-address": localhostIP.String(),
"secure-port": "0",
2019-05-29 18:53:51 +00:00
"use-service-account-credentials": "true",
"cluster-signing-cert-file": runtime.ServerCA,
"cluster-signing-key-file": runtime.ServerCAKey,
2019-01-01 08:23:01 +00:00
}
2019-01-22 21:14:58 +00:00
if cfg.NoLeaderElect {
argsMap["leader-elect"] = "false"
2019-01-22 21:14:58 +00:00
}
args := config.GetArgsList(argsMap, cfg.ExtraControllerArgs)
2019-01-01 08:23:01 +00:00
command := cmapp.NewControllerManagerCommand()
command.SetArgs(args)
go func() {
2019-01-09 16:54:15 +00:00
logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
2019-01-01 08:23:01 +00:00
logrus.Fatalf("controller-manager exited: %v", command.Execute())
}()
}
2019-01-09 16:54:15 +00:00
func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
argsMap := map[string]string{
2019-05-29 18:53:51 +00:00
"kubeconfig": runtime.KubeConfigScheduler,
"port": "10251",
"bind-address": "127.0.0.1",
"secure-port": "0",
2019-01-01 08:23:01 +00:00
}
2019-01-22 21:14:58 +00:00
if cfg.NoLeaderElect {
argsMap["leader-elect"] = "false"
2019-01-22 21:14:58 +00:00
}
args := config.GetArgsList(argsMap, cfg.ExtraSchedulerAPIArgs)
2019-01-01 08:23:01 +00:00
command := sapp.NewSchedulerCommand()
command.SetArgs(args)
go func() {
2019-01-09 16:54:15 +00:00
logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
2019-01-01 08:23:01 +00:00
logrus.Fatalf("scheduler exited: %v", command.Execute())
}()
}
2019-01-09 16:54:15 +00:00
func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
argsMap := make(map[string]string)
2019-06-11 22:48:47 +00:00
setupStorageBackend(argsMap, cfg)
2019-01-01 08:23:01 +00:00
certDir := filepath.Join(cfg.DataDir, "tls/temporary-certs")
os.MkdirAll(certDir, 0700)
argsMap["cert-dir"] = certDir
argsMap["allow-privileged"] = "true"
argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")
argsMap["service-account-signing-key-file"] = runtime.ServiceKey
argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String()
argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort)
2019-05-29 18:53:51 +00:00
if cfg.AdvertiseIP != "" {
argsMap["advertise-address"] = cfg.AdvertiseIP
}
argsMap["insecure-port"] = "0"
argsMap["secure-port"] = strconv.Itoa(cfg.ListenPort)
argsMap["bind-address"] = localhostIP.String()
2019-05-29 18:53:51 +00:00
argsMap["tls-cert-file"] = runtime.ServingKubeAPICert
argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey
argsMap["service-account-key-file"] = runtime.ServiceKey
argsMap["service-account-issuer"] = "k3s"
argsMap["api-audiences"] = "unknown"
argsMap["basic-auth-file"] = runtime.PasswdFile
argsMap["kubelet-certificate-authority"] = runtime.ServerCA
2019-05-29 18:53:51 +00:00
argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert
argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey
argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA
argsMap["requestheader-allowed-names"] = requestHeaderCN
argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert
argsMap["proxy-client-key-file"] = runtime.ClientAuthProxyKey
argsMap["requestheader-extra-headers-prefix"] = "X-Remote-Extra-"
argsMap["requestheader-group-headers"] = "X-Remote-Group"
argsMap["requestheader-username-headers"] = "X-Remote-User"
2019-05-29 18:53:51 +00:00
argsMap["client-ca-file"] = runtime.ClientCA
argsMap["enable-admission-plugins"] = "NodeRestriction"
argsMap["anonymous-auth"] = "false"
args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs)
2019-01-01 08:23:01 +00:00
command := app.NewAPIServerCommand(ctx.Done())
command.SetArgs(args)
go func() {
2019-01-09 16:54:15 +00:00
logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
2019-01-01 08:23:01 +00:00
logrus.Fatalf("apiserver exited: %v", command.Execute())
}()
startupConfig := <-app.StartupConfig
return startupConfig.Authenticator, startupConfig.Handler, nil
}
func defaults(config *config.Control) {
if config.ClusterIPRange == nil {
_, clusterIPNet, _ := net.ParseCIDR("10.42.0.0/16")
config.ClusterIPRange = clusterIPNet
}
if config.ServiceIPRange == nil {
_, serviceIPNet, _ := net.ParseCIDR("10.43.0.0/16")
config.ServiceIPRange = serviceIPNet
}
2019-01-09 16:54:15 +00:00
if len(config.ClusterDNS) == 0 {
config.ClusterDNS = net.ParseIP("10.43.0.10")
}
2019-01-01 08:23:01 +00:00
if config.AdvertisePort == 0 {
2019-05-29 18:53:51 +00:00
config.AdvertisePort = config.HTTPSPort
2019-01-01 08:23:01 +00:00
}
if config.ListenPort == 0 {
if config.HTTPSPort != 0 {
config.ListenPort = config.HTTPSPort + 1
} else {
config.ListenPort = 6444
}
2019-01-01 08:23:01 +00:00
}
if config.DataDir == "" {
config.DataDir = "./management-state"
}
}
func prepare(ctx context.Context, config *config.Control, runtime *config.ControlRuntime) error {
2019-01-01 08:23:01 +00:00
var err error
defaults(config)
if err := os.MkdirAll(config.DataDir, 0700); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
config.DataDir, err = filepath.Abs(config.DataDir)
if err != nil {
return err
}
os.MkdirAll(path.Join(config.DataDir, "tls"), 0700)
os.MkdirAll(path.Join(config.DataDir, "cred"), 0700)
2019-05-29 18:53:51 +00:00
runtime.ClientCA = path.Join(config.DataDir, "tls", "client-ca.crt")
runtime.ClientCAKey = path.Join(config.DataDir, "tls", "client-ca.key")
runtime.ServerCA = path.Join(config.DataDir, "tls", "server-ca.crt")
runtime.ServerCAKey = path.Join(config.DataDir, "tls", "server-ca.key")
runtime.RequestHeaderCA = path.Join(config.DataDir, "tls", "request-header-ca.crt")
runtime.RequestHeaderCAKey = path.Join(config.DataDir, "tls", "request-header-ca.key")
2019-09-03 23:41:54 +00:00
runtime.IPSECKey = path.Join(config.DataDir, "cred", "ipsec.psk")
2019-05-29 18:53:51 +00:00
runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
2019-06-25 17:42:15 +00:00
runtime.NodePasswdFile = path.Join(config.DataDir, "cred", "node-passwd")
2019-05-29 18:53:51 +00:00
runtime.KubeConfigAdmin = path.Join(config.DataDir, "cred", "admin.kubeconfig")
runtime.KubeConfigController = path.Join(config.DataDir, "cred", "controller.kubeconfig")
runtime.KubeConfigScheduler = path.Join(config.DataDir, "cred", "scheduler.kubeconfig")
runtime.KubeConfigAPIServer = path.Join(config.DataDir, "cred", "api-server.kubeconfig")
2019-10-15 21:17:26 +00:00
runtime.KubeConfigCloudController = path.Join(config.DataDir, "cred", "cloud-controller.kubeconfig")
2019-05-29 18:53:51 +00:00
runtime.ClientAdminCert = path.Join(config.DataDir, "tls", "client-admin.crt")
runtime.ClientAdminKey = path.Join(config.DataDir, "tls", "client-admin.key")
runtime.ClientControllerCert = path.Join(config.DataDir, "tls", "client-controller.crt")
runtime.ClientControllerKey = path.Join(config.DataDir, "tls", "client-controller.key")
2019-10-15 21:17:26 +00:00
runtime.ClientCloudControllerCert = path.Join(config.DataDir, "tls", "client-cloud-controller.crt")
runtime.ClientCloudControllerKey = path.Join(config.DataDir, "tls", "client-cloud-controller.key")
2019-05-29 18:53:51 +00:00
runtime.ClientSchedulerCert = path.Join(config.DataDir, "tls", "client-scheduler.crt")
runtime.ClientSchedulerKey = path.Join(config.DataDir, "tls", "client-scheduler.key")
runtime.ClientKubeAPICert = path.Join(config.DataDir, "tls", "client-kube-apiserver.crt")
runtime.ClientKubeAPIKey = path.Join(config.DataDir, "tls", "client-kube-apiserver.key")
runtime.ClientKubeProxyCert = path.Join(config.DataDir, "tls", "client-kube-proxy.crt")
runtime.ClientKubeProxyKey = path.Join(config.DataDir, "tls", "client-kube-proxy.key")
runtime.ClientK3sControllerCert = path.Join(config.DataDir, "tls", "client-k3s-controller.crt")
runtime.ClientK3sControllerKey = path.Join(config.DataDir, "tls", "client-k3s-controller.key")
2019-05-29 18:53:51 +00:00
runtime.ServingKubeAPICert = path.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
runtime.ServingKubeAPIKey = path.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
runtime.ClientKubeletKey = path.Join(config.DataDir, "tls", "client-kubelet.key")
runtime.ServingKubeletKey = path.Join(config.DataDir, "tls", "serving-kubelet.key")
runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
2019-05-29 18:53:51 +00:00
runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
2019-01-01 08:23:01 +00:00
cluster := cluster.New(config)
if err := cluster.Join(ctx); err != nil {
2019-06-27 19:00:43 +00:00
return err
}
if err := genCerts(config, runtime); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
if err := genServiceAccount(runtime); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
if err := genUsers(config, runtime); err != nil {
return err
}
2019-09-03 23:41:54 +00:00
if err := genEncryptedNetworkInfo(config, runtime); err != nil {
return err
}
if err := readTokens(runtime); err != nil {
return err
}
return cluster.Start(ctx)
2019-05-29 18:53:51 +00:00
}
2019-01-01 08:23:01 +00:00
2019-05-29 18:53:51 +00:00
func readTokens(runtime *config.ControlRuntime) error {
tokens, err := passwd.Read(runtime.PasswdFile)
2019-05-29 18:53:51 +00:00
if err != nil {
return err
}
if nodeToken, ok := tokens.Pass("node"); ok {
runtime.AgentToken = "node:" + nodeToken
}
if serverToken, ok := tokens.Pass("server"); ok {
2019-11-08 20:58:50 +00:00
runtime.ServerToken = "server:" + serverToken
2019-05-29 18:53:51 +00:00
}
if clientToken, ok := tokens.Pass("admin"); ok {
2019-05-29 18:53:51 +00:00
runtime.ClientToken = "admin:" + clientToken
2019-01-01 08:23:01 +00:00
}
return nil
}
2019-09-03 23:41:54 +00:00
func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
psk, err := ioutil.ReadFile(runtime.IPSECKey)
if err != nil {
return err
}
controlConfig.IPSECPSK = strings.TrimSpace(string(psk))
return nil
}
psk, err := token.Random(ipsecTokenSize)
2019-09-03 23:41:54 +00:00
if err != nil {
return err
}
controlConfig.IPSECPSK = psk
if err := ioutil.WriteFile(runtime.IPSECKey, []byte(psk+"\n"), 0600); err != nil {
return err
}
return nil
}
func migratePassword(p *passwd.Passwd) error {
server, _ := p.Pass("server")
node, _ := p.Pass("node")
if server == "" && node != "" {
return p.EnsureUser("server", "k3s:server", node)
2019-01-01 08:23:01 +00:00
}
return nil
}
2019-01-01 08:23:01 +00:00
func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) {
var (
err error
)
serverPass := config.Token
if serverPass == "" {
serverPass, _ = passwd.Pass("server")
}
if serverPass == "" {
serverPass, err = token.Random(16)
if err != nil {
return "", err
}
}
return serverPass, nil
}
func getNodePass(config *config.Control, serverPass string) string {
if config.AgentToken == "" {
2019-11-08 20:58:50 +00:00
if _, passwd, ok := clientaccess.ParseUsernamePassword(serverPass); ok {
return passwd
}
return serverPass
}
return config.AgentToken
}
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
passwd, err := passwd.Read(runtime.PasswdFile)
2019-01-01 08:23:01 +00:00
if err != nil {
return err
}
if err := migratePassword(passwd); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
serverPass, err := getServerPass(passwd, config)
2019-01-01 08:23:01 +00:00
if err != nil {
return err
}
nodePass := getNodePass(config, serverPass)
if err := passwd.EnsureUser("admin", "system:masters", ""); err != nil {
return err
2019-01-22 21:14:58 +00:00
}
if err := passwd.EnsureUser("node", "k3s:agent", nodePass); err != nil {
return err
}
2019-01-01 08:23:01 +00:00
if err := passwd.EnsureUser("server", "k3s:server", serverPass); err != nil {
return err
2019-01-01 08:23:01 +00:00
}
return passwd.Write(runtime.PasswdFile)
2019-01-01 08:23:01 +00:00
}
func genCerts(config *config.Control, runtime *config.ControlRuntime) error {
2019-05-29 18:53:51 +00:00
if err := genClientCerts(config, runtime); err != nil {
return err
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
if err := genServerCerts(config, runtime); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
if err := genRequestHeaderCerts(config, runtime); err != nil {
return err
2019-01-01 08:23:01 +00:00
}
return nil
}
2019-01-01 08:23:01 +00:00
2019-05-29 18:53:51 +00:00
type signedCertFactory = func(commonName string, organization []string, certFile, keyFile string) (bool, error)
func getSigningCertFactory(regen bool, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile string) signedCertFactory {
return func(commonName string, organization []string, certFile, keyFile string) (bool, error) {
return createClientCertKey(regen, commonName, organization, altNames, extKeyUsage, caCertFile, caKeyFile, certFile, keyFile)
}
}
func genClientCerts(config *config.Control, runtime *config.ControlRuntime) error {
regen, err := createSigningCertKey("k3s-client", runtime.ClientCA, runtime.ClientCAKey)
if err != nil {
return err
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
factory := getSigningCertFactory(regen, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.ClientCA, runtime.ClientCAKey)
var certGen bool
2019-08-22 18:56:00 +00:00
apiEndpoint := fmt.Sprintf("https://127.0.0.1:%d", config.ListenPort)
2019-05-29 18:53:51 +00:00
certGen, err = factory("system:admin", []string{"system:masters"}, runtime.ClientAdminCert, runtime.ClientAdminKey)
2019-01-01 08:23:01 +00:00
if err != nil {
return err
}
2019-05-29 18:53:51 +00:00
if certGen {
if err := KubeConfig(runtime.KubeConfigAdmin, apiEndpoint, runtime.ServerCA, runtime.ClientAdminCert, runtime.ClientAdminKey); err != nil {
return err
}
}
2019-01-01 08:23:01 +00:00
2019-05-29 18:53:51 +00:00
certGen, err = factory("system:kube-controller-manager", nil, runtime.ClientControllerCert, runtime.ClientControllerKey)
if err != nil {
return err
}
if certGen {
if err := KubeConfig(runtime.KubeConfigController, apiEndpoint, runtime.ServerCA, runtime.ClientControllerCert, runtime.ClientControllerKey); err != nil {
return err
}
}
certGen, err = factory("system:kube-scheduler", nil, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey)
if err != nil {
return err
}
if certGen {
if err := KubeConfig(runtime.KubeConfigScheduler, apiEndpoint, runtime.ServerCA, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey); err != nil {
return err
}
}
certGen, err = factory("kube-apiserver", nil, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey)
if err != nil {
return err
}
if certGen {
if err := KubeConfig(runtime.KubeConfigAPIServer, apiEndpoint, runtime.ServerCA, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey); err != nil {
return err
}
}
if _, err = factory("system:kube-proxy", nil, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
return err
}
if _, err = factory("system:k3s-controller", nil, runtime.ClientK3sControllerCert, runtime.ClientK3sControllerKey); err != nil {
2019-05-29 18:53:51 +00:00
return err
}
2019-09-26 22:38:04 +00:00
if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ClientKubeletKey, regen); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
2019-10-15 21:17:26 +00:00
certGen, err = factory("cloud-controller-manager", nil, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey)
if err != nil {
return err
}
if certGen {
if err := KubeConfig(runtime.KubeConfigCloudController, apiEndpoint, runtime.ServerCA, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey); err != nil {
return err
}
}
return nil
}
2019-06-21 17:45:31 +00:00
func createServerSigningCertKey(config *config.Control, runtime *config.ControlRuntime) (bool, error) {
TokenCA := path.Join(config.DataDir, "tls", "token-ca.crt")
TokenCAKey := path.Join(config.DataDir, "tls", "token-ca.key")
if exists(TokenCA, TokenCAKey) && !exists(runtime.ServerCA) && !exists(runtime.ServerCAKey) {
logrus.Infof("Upgrading token-ca files to server-ca")
if err := os.Link(TokenCA, runtime.ServerCA); err != nil {
return false, err
}
if err := os.Link(TokenCAKey, runtime.ServerCAKey); err != nil {
return false, err
}
return true, nil
}
return createSigningCertKey("k3s-server", runtime.ServerCA, runtime.ServerCAKey)
}
2019-05-29 18:53:51 +00:00
func genServerCerts(config *config.Control, runtime *config.ControlRuntime) error {
2019-06-21 17:45:31 +00:00
regen, err := createServerSigningCertKey(config, runtime)
2019-01-01 08:23:01 +00:00
if err != nil {
return err
}
_, apiServerServiceIP, err := master.DefaultServiceIPRange(*config.ServiceIPRange)
2019-01-01 08:23:01 +00:00
if err != nil {
return err
}
2019-05-29 18:53:51 +00:00
if _, err := createClientCertKey(regen, "kube-apiserver", nil,
&certutil.AltNames{
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
IPs: []net.IP{apiServerServiceIP, localhostIP},
2019-05-29 18:53:51 +00:00
}, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
runtime.ServerCA, runtime.ServerCAKey,
runtime.ServingKubeAPICert, runtime.ServingKubeAPIKey); err != nil {
return err
}
2019-09-26 22:38:04 +00:00
if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ServingKubeletKey, regen); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
return nil
}
func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntime) error {
regen, err := createSigningCertKey("k3s-request-header", runtime.RequestHeaderCA, runtime.RequestHeaderCAKey)
2019-01-01 08:23:01 +00:00
if err != nil {
return err
}
2019-05-29 18:53:51 +00:00
if _, err := createClientCertKey(regen, requestHeaderCN, nil,
nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
runtime.RequestHeaderCA, runtime.RequestHeaderCAKey,
runtime.ClientAuthProxyCert, runtime.ClientAuthProxyKey); err != nil {
2019-01-01 08:23:01 +00:00
return err
}
return nil
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) (bool, error) {
2019-11-08 20:58:50 +00:00
caBytes, err := ioutil.ReadFile(caCertFile)
if err != nil {
return false, err
}
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(caBytes)
2019-09-26 22:38:04 +00:00
// check for certificate expiration
if !regen {
2019-11-08 20:58:50 +00:00
regen = expired(certFile, pool)
2019-09-26 22:38:04 +00:00
}
2019-01-01 08:23:01 +00:00
if !regen {
if exists(certFile, keyFile) {
2019-05-29 18:53:51 +00:00
return false, nil
2019-01-01 08:23:01 +00:00
}
}
caKeyBytes, err := ioutil.ReadFile(caKeyFile)
2019-01-01 08:23:01 +00:00
if err != nil {
2019-05-29 18:53:51 +00:00
return false, err
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
2019-01-01 08:23:01 +00:00
if err != nil {
2019-05-29 18:53:51 +00:00
return false, err
2019-01-01 08:23:01 +00:00
}
caCert, err := certutil.ParseCertsPEM(caBytes)
if err != nil {
2019-05-29 18:53:51 +00:00
return false, err
2019-01-01 08:23:01 +00:00
}
2019-11-08 20:58:50 +00:00
2019-09-26 22:38:04 +00:00
keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, regen)
2019-01-01 08:23:01 +00:00
if err != nil {
2019-05-29 18:53:51 +00:00
return false, err
}
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
if err != nil {
return false, err
2019-01-01 08:23:01 +00:00
}
cfg := certutil.Config{
CommonName: commonName,
Organization: organization,
Usages: extKeyUsage,
}
if altNames != nil {
cfg.AltNames = *altNames
}
2019-05-29 18:53:51 +00:00
cert, err := certutil.NewSignedCert(cfg, key.(crypto.Signer), caCert[0], caKey.(crypto.Signer))
2019-01-01 08:23:01 +00:00
if err != nil {
2019-05-29 18:53:51 +00:00
return false, err
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
return true, certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
}
func exists(files ...string) bool {
for _, file := range files {
if _, err := os.Stat(file); err != nil {
return false
}
}
return true
2019-01-01 08:23:01 +00:00
}
func genServiceAccount(runtime *config.ControlRuntime) error {
_, keyErr := os.Stat(runtime.ServiceKey)
if keyErr == nil {
return nil
}
key, err := certutil.NewPrivateKey()
if err != nil {
return err
}
return certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key))
}
func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) {
if exists(certFile, keyFile) {
return false, nil
2019-01-01 08:23:01 +00:00
}
2019-09-26 22:38:04 +00:00
caKeyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
2019-05-29 18:53:51 +00:00
if err != nil {
return false, err
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
2019-01-01 08:23:01 +00:00
if err != nil {
return false, err
2019-01-01 08:23:01 +00:00
}
cfg := certutil.Config{
CommonName: fmt.Sprintf("%s-ca@%d", prefix, time.Now().Unix()),
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
cert, err := certutil.NewSelfSignedCACert(cfg, caKey.(crypto.Signer))
2019-01-01 08:23:01 +00:00
if err != nil {
return false, err
2019-01-01 08:23:01 +00:00
}
if err := certutil.WriteCert(certFile, certutil.EncodeCertPEM(cert)); err != nil {
return false, err
}
return true, nil
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
func KubeConfig(dest, url, caCert, clientCert, clientKey string) error {
2019-01-01 08:23:01 +00:00
data := struct {
2019-05-29 18:53:51 +00:00
URL string
CACert string
ClientCert string
ClientKey string
2019-01-01 08:23:01 +00:00
}{
2019-05-29 18:53:51 +00:00
URL: url,
CACert: caCert,
ClientCert: clientCert,
ClientKey: clientKey,
2019-01-01 08:23:01 +00:00
}
output, err := os.Create(dest)
if err != nil {
return err
}
defer output.Close()
return kubeconfigTemplate.Execute(output, &data)
}
2019-06-11 22:48:47 +00:00
func setupStorageBackend(argsMap map[string]string, cfg *config.Control) {
argsMap["storage-backend"] = "etcd3"
2019-06-11 22:48:47 +00:00
// specify the endpoints
2019-11-16 00:12:27 +00:00
if len(cfg.Datastore.Endpoint) > 0 {
argsMap["etcd-servers"] = cfg.Datastore.Endpoint
2019-06-11 22:48:47 +00:00
}
// storage backend tls configuration
2019-11-16 00:12:27 +00:00
if len(cfg.Datastore.CAFile) > 0 {
argsMap["etcd-cafile"] = cfg.Datastore.CAFile
2019-06-11 22:48:47 +00:00
}
2019-11-16 00:12:27 +00:00
if len(cfg.Datastore.CertFile) > 0 {
argsMap["etcd-certfile"] = cfg.Datastore.CertFile
2019-06-11 22:48:47 +00:00
}
2019-11-16 00:12:27 +00:00
if len(cfg.Datastore.KeyFile) > 0 {
argsMap["etcd-keyfile"] = cfg.Datastore.KeyFile
2019-06-11 22:48:47 +00:00
}
}
2019-09-26 22:38:04 +00:00
2019-11-08 20:58:50 +00:00
func expired(certFile string, pool *x509.CertPool) bool {
2019-09-26 22:38:04 +00:00
certBytes, err := ioutil.ReadFile(certFile)
if err != nil {
return false
}
certificates, err := certutil.ParseCertsPEM(certBytes)
if err != nil {
return false
}
2019-11-08 20:58:50 +00:00
_, err = certificates[0].Verify(x509.VerifyOptions{
Roots: pool,
KeyUsages: []x509.ExtKeyUsage{
x509.ExtKeyUsageAny,
},
})
if err != nil {
return true
}
2019-10-10 01:35:32 +00:00
return certutil.IsCertExpired(certificates[0])
2019-09-26 22:38:04 +00:00
}
2019-10-15 21:17:26 +00:00
func cloudControllerManager(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) {
2019-10-15 21:17:26 +00:00
argsMap := map[string]string{
"kubeconfig": runtime.KubeConfigCloudController,
"allocate-node-cidrs": "true",
"cluster-cidr": cfg.ClusterIPRange.String(),
"bind-address": localhostIP.String(),
"secure-port": "0",
"cloud-provider": "k3s",
"allow-untagged-cloud": "true",
"node-status-update-frequency": "1m",
}
if cfg.NoLeaderElect {
argsMap["leader-elect"] = "false"
}
args := config.GetArgsList(argsMap, cfg.ExtraCloudControllerArgs)
command := ccmapp.NewCloudControllerManagerCommand()
command.SetArgs(args)
// register k3s cloud provider
go func() {
for {
// check for the cloud controller rbac binding
if err := checkForCloudControllerPrivileges(runtime); err != nil {
logrus.Infof("Waiting for cloudcontroller rbac role to be created")
select {
case <-ctx.Done():
logrus.Fatalf("cloud-controller-manager context canceled: %v", ctx.Err())
case <-time.After(time.Second):
continue
}
2019-10-15 21:17:26 +00:00
}
break
}
logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args))
logrus.Fatalf("cloud-controller-manager exited: %v", command.Execute())
}()
}
func checkForCloudControllerPrivileges(runtime *config.ControlRuntime) error {
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
if err != nil {
return err
}
crb := rbac.NewFactoryFromConfigOrDie(restConfig).Rbac().V1().ClusterRoleBinding()
_, err = crb.Get("cloud-controller-manager", metav1.GetOptions{})
if err != nil {
return err
}
return nil
}
func waitForAPIServer(ctx context.Context, runtime *config.ControlRuntime) error {
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
if err != nil {
return err
}
k8sClient, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 5*time.Minute) }):
return err
}
}
func promise(f func() error) <-chan error {
c := make(chan error, 1)
go func() {
c <- f()
close(c)
}()
return c
}