k3s/pkg/daemons/control/server.go

608 lines
15 KiB
Go
Raw Normal View History

2019-01-01 08:23:01 +00:00
package control
import (
2019-01-22 21:14:58 +00:00
"bufio"
2019-01-01 08:23:01 +00:00
"context"
cryptorand "crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/csv"
"encoding/hex"
"fmt"
"html/template"
"io"
"io/ioutil"
"math/rand"
"net"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/daemons/config"
2019-01-01 08:23:01 +00:00
"github.com/sirupsen/logrus"
"k8s.io/apiserver/pkg/authentication/authenticator"
certutil "k8s.io/client-go/util/cert"
"k8s.io/kubernetes/cmd/kube-apiserver/app"
cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
sapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
"k8s.io/kubernetes/pkg/master"
2019-02-08 04:14:02 +00:00
"k8s.io/kubernetes/pkg/proxy/util"
2019-01-01 08:23:01 +00:00
_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
)
var (
localhostIP = net.ParseIP("127.0.0.1")
kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
clusters:
- cluster:
server: {{.URL}}
certificate-authority-data: {{.CACert}}
name: local
contexts:
- context:
cluster: local
namespace: default
user: user
name: Default
current-context: Default
kind: Config
preferences: {}
users:
- name: user
user:
username: {{.User}}
password: {{.Password}}
`))
)
func Server(ctx context.Context, cfg *config.Control) error {
rand.Seed(time.Now().UTC().UnixNano())
runtime := &config.ControlRuntime{}
cfg.Runtime = runtime
if err := prepare(cfg, runtime); err != nil {
return err
}
cfg.Runtime.Tunnel = setupTunnel()
2019-02-08 04:14:02 +00:00
util.DisableProxyHostnameCheck = true
2019-01-01 08:23:01 +00:00
auth, handler, err := apiServer(ctx, cfg, runtime)
if err != nil {
return err
}
runtime.Handler = handler
runtime.Authenticator = auth
if !cfg.NoScheduler {
scheduler(cfg, runtime)
}
controllerManager(cfg, runtime)
return nil
}
2019-01-09 16:54:15 +00:00
func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
2019-01-01 08:23:01 +00:00
args := []string{
"--kubeconfig", runtime.KubeConfigSystem,
"--service-account-private-key-file", runtime.ServiceKey,
"--allocate-node-cidrs",
2019-01-09 16:54:15 +00:00
"--cluster-cidr", cfg.ClusterIPRange.String(),
2019-01-01 08:23:01 +00:00
"--root-ca-file", runtime.TokenCA,
"--port", "10252",
"--bind-address", "127.0.0.1",
"--secure-port", "0",
2019-01-01 08:23:01 +00:00
}
2019-01-22 21:14:58 +00:00
if cfg.NoLeaderElect {
args = append(args, "--leader-elect=false")
}
2019-01-09 16:54:15 +00:00
args = append(args, cfg.ExtraControllerArgs...)
2019-01-01 08:23:01 +00:00
command := cmapp.NewControllerManagerCommand()
command.SetArgs(args)
go func() {
2019-01-09 16:54:15 +00:00
logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
2019-01-01 08:23:01 +00:00
logrus.Fatalf("controller-manager exited: %v", command.Execute())
}()
}
2019-01-09 16:54:15 +00:00
func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
2019-01-01 08:23:01 +00:00
args := []string{
"--kubeconfig", runtime.KubeConfigSystem,
"--port", "10251",
"--bind-address", "127.0.0.1",
"--secure-port", "0",
2019-01-01 08:23:01 +00:00
}
2019-01-22 21:14:58 +00:00
if cfg.NoLeaderElect {
args = append(args, "--leader-elect=false")
}
2019-01-09 16:54:15 +00:00
args = append(args, cfg.ExtraSchedulerAPIArgs...)
2019-01-01 08:23:01 +00:00
command := sapp.NewSchedulerCommand()
command.SetArgs(args)
go func() {
2019-01-09 16:54:15 +00:00
logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
2019-01-01 08:23:01 +00:00
logrus.Fatalf("scheduler exited: %v", command.Execute())
}()
}
2019-01-09 16:54:15 +00:00
func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
2019-01-01 08:23:01 +00:00
var args []string
2019-01-09 16:54:15 +00:00
if len(cfg.ETCDEndpoints) > 0 {
2019-01-01 08:23:01 +00:00
args = append(args, "--storage-backend", "etcd3")
2019-01-09 16:54:15 +00:00
args = append(args, "--etcd-servers", strings.Join(cfg.ETCDEndpoints, ","))
if cfg.ETCDKeyFile != "" {
args = append(args, "--etcd-keyfile", cfg.ETCDKeyFile)
2019-01-01 08:23:01 +00:00
}
2019-01-09 16:54:15 +00:00
if cfg.ETCDCAFile != "" {
args = append(args, "--etcd-cafile", cfg.ETCDCAFile)
2019-01-01 08:23:01 +00:00
}
2019-01-09 16:54:15 +00:00
if cfg.ETCDCertFile != "" {
args = append(args, "--etcd-certfile", cfg.ETCDCertFile)
2019-01-01 08:23:01 +00:00
}
}
certDir := filepath.Join(cfg.DataDir, "tls/temporary-certs")
os.MkdirAll(certDir, 0700)
// TODO: sqlite doesn't need the watch cache, but etcd does, so make this dynamic
args = append(args, "--watch-cache=false")
args = append(args, "--cert-dir", certDir)
2019-01-01 08:23:01 +00:00
args = append(args, "--allow-privileged=true")
args = append(args, "--authorization-mode", strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ","))
args = append(args, "--service-account-signing-key-file", runtime.ServiceKey)
2019-01-09 16:54:15 +00:00
args = append(args, "--service-cluster-ip-range", cfg.ServiceIPRange.String())
args = append(args, "--advertise-port", strconv.Itoa(cfg.AdvertisePort))
2019-01-01 08:23:01 +00:00
args = append(args, "--advertise-address", localhostIP.String())
args = append(args, "--insecure-port", "0")
2019-01-09 16:54:15 +00:00
args = append(args, "--secure-port", strconv.Itoa(cfg.ListenPort))
2019-01-01 08:23:01 +00:00
args = append(args, "--bind-address", localhostIP.String())
args = append(args, "--tls-cert-file", runtime.TLSCert)
args = append(args, "--tls-private-key-file", runtime.TLSKey)
args = append(args, "--service-account-key-file", runtime.ServiceKey)
args = append(args, "--service-account-issuer", "k3s")
args = append(args, "--api-audiences", "unknown")
args = append(args, "--basic-auth-file", runtime.PasswdFile)
2019-01-09 16:54:15 +00:00
args = append(args, "--kubelet-client-certificate", runtime.NodeCert)
args = append(args, "--kubelet-client-key", runtime.NodeKey)
2019-01-01 08:23:01 +00:00
2019-01-09 16:54:15 +00:00
args = append(args, cfg.ExtraAPIArgs...)
2019-01-01 08:23:01 +00:00
command := app.NewAPIServerCommand(ctx.Done())
command.SetArgs(args)
go func() {
2019-01-09 16:54:15 +00:00
logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
2019-01-01 08:23:01 +00:00
logrus.Fatalf("apiserver exited: %v", command.Execute())
}()
startupConfig := <-app.StartupConfig
return startupConfig.Authenticator, startupConfig.Handler, nil
}
func defaults(config *config.Control) {
if config.ClusterIPRange == nil {
_, clusterIPNet, _ := net.ParseCIDR("10.42.0.0/16")
config.ClusterIPRange = clusterIPNet
}
if config.ServiceIPRange == nil {
_, serviceIPNet, _ := net.ParseCIDR("10.43.0.0/16")
config.ServiceIPRange = serviceIPNet
}
2019-01-09 16:54:15 +00:00
if len(config.ClusterDNS) == 0 {
config.ClusterDNS = net.ParseIP("10.43.0.10")
}
2019-01-01 08:23:01 +00:00
if config.AdvertisePort == 0 {
config.AdvertisePort = 6445
}
if config.ListenPort == 0 {
config.ListenPort = 6444
}
if config.DataDir == "" {
config.DataDir = "./management-state"
}
}
func prepare(config *config.Control, runtime *config.ControlRuntime) error {
var err error
defaults(config)
if _, err := os.Stat(config.DataDir); os.IsNotExist(err) {
if err := os.MkdirAll(config.DataDir, 0700); err != nil {
return err
}
} else if err != nil {
return err
}
config.DataDir, err = filepath.Abs(config.DataDir)
if err != nil {
return err
}
os.MkdirAll(path.Join(config.DataDir, "tls"), 0700)
os.MkdirAll(path.Join(config.DataDir, "cred"), 0700)
name := "localhost"
runtime.TLSCert = path.Join(config.DataDir, "tls", name+".crt")
runtime.TLSKey = path.Join(config.DataDir, "tls", name+".key")
runtime.TLSCA = path.Join(config.DataDir, "tls", "ca.crt")
runtime.TLSCAKey = path.Join(config.DataDir, "tls", "ca.key")
runtime.TokenCA = path.Join(config.DataDir, "tls", "token-ca.crt")
runtime.TokenCAKey = path.Join(config.DataDir, "tls", "token-ca.key")
runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
runtime.KubeConfigSystem = path.Join(config.DataDir, "cred", "kubeconfig-system.yaml")
runtime.NodeKey = path.Join(config.DataDir, "tls", "token-node.key")
runtime.NodeCert = path.Join(config.DataDir, "tls", "token-node.crt")
regen := false
if _, err := os.Stat(runtime.TLSCA); err != nil {
regen = true
if err := genCA(runtime); err != nil {
return err
}
}
if err := genServiceAccount(runtime); err != nil {
return err
}
if err := genTLS(regen, config, runtime); err != nil {
return err
}
if err := genTokenTLS(config, runtime); err != nil {
return err
}
if err := genUsers(config, runtime); err != nil {
return err
}
return readTokens(runtime)
}
func readTokens(runtime *config.ControlRuntime) error {
f, err := os.Open(runtime.PasswdFile)
if err != nil {
return err
}
reader := csv.NewReader(f)
reader.FieldsPerRecord = -1
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
if len(record) < 2 {
continue
}
switch record[1] {
case "node":
runtime.NodeToken = "node:" + record[0]
case "admin":
runtime.ClientToken = "admin:" + record[0]
}
}
return nil
}
2019-01-22 21:14:58 +00:00
func ensureNodeToken(config *config.Control, runtime *config.ControlRuntime) error {
if config.ClusterSecret == "" {
return nil
}
f, err := os.Open(runtime.PasswdFile)
if err != nil {
return err
}
defer f.Close()
buf := &strings.Builder{}
scan := bufio.NewScanner(f)
for scan.Scan() {
line := scan.Text()
parts := strings.Split(line, ",")
if len(parts) < 4 {
continue
}
if parts[1] == "node" {
if parts[0] == config.ClusterSecret {
return nil
}
parts[0] = config.ClusterSecret
line = strings.Join(parts, ",")
}
buf.WriteString(line)
buf.WriteString("\n")
}
if scan.Err() != nil {
return scan.Err()
}
f.Close()
return ioutil.WriteFile(runtime.PasswdFile, []byte(buf.String()), 0600)
}
2019-01-01 08:23:01 +00:00
func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
2019-01-22 21:14:58 +00:00
return ensureNodeToken(config, runtime)
2019-01-01 08:23:01 +00:00
}
adminToken, err := getToken()
if err != nil {
return err
}
systemToken, err := getToken()
if err != nil {
return err
}
nodeToken, err := getToken()
if err != nil {
return err
}
2019-01-22 21:14:58 +00:00
if config.ClusterSecret != "" {
nodeToken = config.ClusterSecret
}
2019-01-01 08:23:01 +00:00
passwd := fmt.Sprintf(`%s,admin,admin,system:masters
%s,system,system,system:masters
%s,node,node,system:masters
`, adminToken, systemToken, nodeToken)
caCertBytes, err := ioutil.ReadFile(runtime.TLSCA)
if err != nil {
return err
}
caCert := base64.StdEncoding.EncodeToString(caCertBytes)
if err := kubeConfig(runtime.KubeConfigSystem, fmt.Sprintf("https://localhost:%d", config.ListenPort), caCert,
"system", systemToken); err != nil {
return err
}
return ioutil.WriteFile(runtime.PasswdFile, []byte(passwd), 0600)
}
func getToken() (string, error) {
token := make([]byte, 16, 16)
_, err := cryptorand.Read(token)
if err != nil {
return "", err
}
return hex.EncodeToString(token), err
}
func genTokenTLS(config *config.Control, runtime *config.ControlRuntime) error {
regen := false
if _, err := os.Stat(runtime.TokenCA); err != nil {
regen = true
if err := genTokenCA(runtime); err != nil {
return err
}
}
_, apiServerServiceIP, err := master.DefaultServiceIPRange(*config.ServiceIPRange)
if err != nil {
return err
}
cfg := certutil.Config{
CommonName: "kubernetes",
AltNames: certutil.AltNames{
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
IPs: []net.IP{net.ParseIP("127.0.0.1"), apiServerServiceIP},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
}
if _, err := os.Stat(runtime.NodeCert); err == nil && !regen {
return nil
}
caKeyBytes, err := ioutil.ReadFile(runtime.TokenCAKey)
if err != nil {
return err
}
caBytes, err := ioutil.ReadFile(runtime.TokenCA)
if err != nil {
return err
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
if err != nil {
return err
}
caCert, err := certutil.ParseCertsPEM(caBytes)
if err != nil {
return err
}
key, err := certutil.NewPrivateKey()
if err != nil {
return err
}
cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey))
if err != nil {
return err
}
if err := certutil.WriteKey(runtime.NodeKey, certutil.EncodePrivateKeyPEM(key)); err != nil {
return err
}
return certutil.WriteCert(runtime.NodeCert, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
}
func genTLS(regen bool, config *config.Control, runtime *config.ControlRuntime) error {
if !regen {
_, certErr := os.Stat(runtime.TLSCert)
_, keyErr := os.Stat(runtime.TLSKey)
if certErr == nil && keyErr == nil {
return nil
}
}
_, apiServerServiceIP, err := master.DefaultServiceIPRange(*config.ServiceIPRange)
if err != nil {
return err
}
cfg := certutil.Config{
CommonName: "localhost",
AltNames: certutil.AltNames{
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
IPs: []net.IP{apiServerServiceIP, localhostIP},
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
}
caKeyBytes, err := ioutil.ReadFile(runtime.TLSCAKey)
if err != nil {
return err
}
caBytes, err := ioutil.ReadFile(runtime.TLSCA)
if err != nil {
return err
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
if err != nil {
return err
}
caCert, err := certutil.ParseCertsPEM(caBytes)
if err != nil {
return err
}
key, err := certutil.NewPrivateKey()
if err != nil {
return err
}
cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey))
if err != nil {
return err
}
if err := certutil.WriteKey(runtime.TLSKey, certutil.EncodePrivateKeyPEM(key)); err != nil {
return err
}
return certutil.WriteCert(runtime.TLSCert, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
}
func genServiceAccount(runtime *config.ControlRuntime) error {
_, keyErr := os.Stat(runtime.ServiceKey)
if keyErr == nil {
return nil
}
key, err := certutil.NewPrivateKey()
if err != nil {
return err
}
return certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key))
}
func genTokenCA(runtime *config.ControlRuntime) error {
caKey, err := certutil.NewPrivateKey()
if err != nil {
return err
}
cfg := certutil.Config{
CommonName: fmt.Sprintf("%s-ca@%d", "k3s-token", time.Now().Unix()),
}
cert, err := certutil.NewSelfSignedCACert(cfg, caKey)
if err != nil {
return err
}
if err := certutil.WriteKey(runtime.TokenCAKey, certutil.EncodePrivateKeyPEM(caKey)); err != nil {
return err
}
return certutil.WriteCert(runtime.TokenCA, certutil.EncodeCertPEM(cert))
}
func genCA(runtime *config.ControlRuntime) error {
caKey, err := certutil.NewPrivateKey()
if err != nil {
return err
}
cfg := certutil.Config{
CommonName: fmt.Sprintf("%s-ca@%d", "k3s", time.Now().Unix()),
}
cert, err := certutil.NewSelfSignedCACert(cfg, caKey)
if err != nil {
return err
}
if err := certutil.WriteKey(runtime.TLSCAKey, certutil.EncodePrivateKeyPEM(caKey)); err != nil {
return err
}
return certutil.WriteCert(runtime.TLSCA, certutil.EncodeCertPEM(cert))
}
func kubeConfig(dest, url, cert, user, password string) error {
data := struct {
URL string
CACert string
User string
Password string
}{
URL: url,
CACert: cert,
User: user,
Password: password,
}
output, err := os.Create(dest)
if err != nil {
return err
}
defer output.Close()
return kubeconfigTemplate.Execute(output, &data)
}