2019-01-01 08:23:01 +00:00
|
|
|
package server
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"crypto/sha256"
|
|
|
|
"encoding/hex"
|
|
|
|
"fmt"
|
|
|
|
"io/ioutil"
|
2019-01-09 16:54:15 +00:00
|
|
|
net2 "net"
|
2019-01-01 08:23:01 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2019-01-22 21:14:58 +00:00
|
|
|
"strconv"
|
2019-01-01 08:23:01 +00:00
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/pkg/errors"
|
2019-01-09 16:54:15 +00:00
|
|
|
"github.com/rancher/k3s/pkg/daemons/config"
|
|
|
|
"github.com/rancher/k3s/pkg/daemons/control"
|
2019-01-22 21:14:58 +00:00
|
|
|
"github.com/rancher/k3s/pkg/deploy"
|
2019-02-04 23:33:23 +00:00
|
|
|
"github.com/rancher/k3s/pkg/helm"
|
2019-02-02 05:09:11 +00:00
|
|
|
"github.com/rancher/k3s/pkg/servicelb"
|
2019-01-09 16:54:15 +00:00
|
|
|
"github.com/rancher/k3s/pkg/tls"
|
2019-02-02 05:09:11 +00:00
|
|
|
appsv1 "github.com/rancher/k3s/types/apis/apps/v1"
|
2019-02-04 23:33:23 +00:00
|
|
|
batchv1 "github.com/rancher/k3s/types/apis/batch/v1"
|
2019-02-02 05:09:11 +00:00
|
|
|
corev1 "github.com/rancher/k3s/types/apis/core/v1"
|
2019-01-09 16:54:15 +00:00
|
|
|
v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1"
|
2019-02-04 23:33:23 +00:00
|
|
|
rbacv1 "github.com/rancher/k3s/types/apis/rbac.authorization.k8s.io/v1"
|
2019-01-01 08:23:01 +00:00
|
|
|
"github.com/rancher/norman"
|
|
|
|
"github.com/rancher/norman/pkg/clientaccess"
|
|
|
|
"github.com/rancher/norman/pkg/dynamiclistener"
|
|
|
|
"github.com/rancher/norman/pkg/resolvehome"
|
|
|
|
"github.com/rancher/norman/types"
|
|
|
|
"github.com/sirupsen/logrus"
|
|
|
|
"k8s.io/apimachinery/pkg/util/net"
|
|
|
|
)
|
|
|
|
|
|
|
|
func resolveDataDir(dataDir string) (string, error) {
|
|
|
|
if dataDir == "" {
|
|
|
|
if os.Getuid() == 0 {
|
|
|
|
dataDir = "/var/lib/rancher/k3s"
|
|
|
|
} else {
|
|
|
|
dataDir = "${HOME}/.rancher/k3s"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
dataDir = filepath.Join(dataDir, "server")
|
|
|
|
return resolvehome.Resolve(dataDir)
|
|
|
|
}
|
|
|
|
|
2019-01-09 16:54:15 +00:00
|
|
|
func StartServer(ctx context.Context, config *Config) (string, error) {
|
2019-01-01 08:23:01 +00:00
|
|
|
if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
|
2019-01-09 16:54:15 +00:00
|
|
|
return "", err
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err := control.Server(ctx, &config.ControlConfig); err != nil {
|
2019-01-09 16:54:15 +00:00
|
|
|
return "", errors.Wrap(err, "starting kubernetes")
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2019-02-02 05:09:11 +00:00
|
|
|
certs, err := startNorman(ctx, config)
|
2019-01-01 08:23:01 +00:00
|
|
|
if err != nil {
|
2019-01-09 16:54:15 +00:00
|
|
|
return "", errors.Wrap(err, "starting tls server")
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2019-01-09 16:54:15 +00:00
|
|
|
ip, err := net.ChooseHostInterface()
|
|
|
|
if err != nil {
|
|
|
|
ip = net2.ParseIP("127.0.0.1")
|
|
|
|
}
|
|
|
|
printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig)
|
2019-01-01 08:23:01 +00:00
|
|
|
|
|
|
|
writeKubeConfig(certs, &config.TLSConfig, &config.ControlConfig)
|
|
|
|
|
2019-01-09 16:54:15 +00:00
|
|
|
return certs, nil
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
2019-02-02 05:09:11 +00:00
|
|
|
func startNorman(ctx context.Context, config *Config) (string, error) {
|
2019-01-01 08:23:01 +00:00
|
|
|
var (
|
2019-02-02 05:09:11 +00:00
|
|
|
err error
|
|
|
|
tlsServer dynamiclistener.ServerInterface
|
|
|
|
tlsConfig = &config.TLSConfig
|
|
|
|
controlConfig = &config.ControlConfig
|
2019-01-01 08:23:01 +00:00
|
|
|
)
|
|
|
|
|
2019-02-02 05:09:11 +00:00
|
|
|
tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
|
2019-01-01 08:23:01 +00:00
|
|
|
if tlsServer == nil {
|
|
|
|
return "", nil
|
|
|
|
}
|
|
|
|
return tlsServer.CACert()
|
|
|
|
})
|
|
|
|
|
|
|
|
normanConfig := &norman.Config{
|
|
|
|
Name: "k3s",
|
2019-02-02 05:09:11 +00:00
|
|
|
KubeConfig: controlConfig.Runtime.KubeConfigSystem,
|
2019-01-01 08:23:01 +00:00
|
|
|
Clients: []norman.ClientFactory{
|
|
|
|
v1.Factory,
|
2019-02-02 05:09:11 +00:00
|
|
|
appsv1.Factory,
|
|
|
|
corev1.Factory,
|
2019-02-04 23:33:23 +00:00
|
|
|
batchv1.Factory,
|
|
|
|
rbacv1.Factory,
|
2019-01-01 08:23:01 +00:00
|
|
|
},
|
|
|
|
Schemas: []*types.Schemas{
|
|
|
|
v1.Schemas,
|
|
|
|
},
|
|
|
|
CRDs: map[*types.APIVersion][]string{
|
|
|
|
&v1.APIVersion: {
|
|
|
|
v1.ListenerConfigGroupVersionKind.Kind,
|
2019-01-22 21:14:58 +00:00
|
|
|
v1.AddonGroupVersionKind.Kind,
|
2019-02-04 23:33:23 +00:00
|
|
|
v1.HelmChartGroupVersionKind.Kind,
|
2019-01-01 08:23:01 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
IgnoredKubeConfigEnv: true,
|
|
|
|
GlobalSetup: func(ctx context.Context) (context.Context, error) {
|
|
|
|
tlsServer, err = tls.NewServer(ctx, v1.ClientsFrom(ctx).ListenerConfig, *tlsConfig)
|
|
|
|
return ctx, err
|
|
|
|
},
|
2019-01-31 22:42:38 +00:00
|
|
|
DisableLeaderElection: true,
|
2019-01-22 21:14:58 +00:00
|
|
|
MasterControllers: []norman.ControllerRegister{
|
2019-02-04 23:33:23 +00:00
|
|
|
helm.Register,
|
2019-01-22 21:14:58 +00:00
|
|
|
func(ctx context.Context) error {
|
2019-02-02 05:09:11 +00:00
|
|
|
return servicelb.Register(ctx, norman.GetServer(ctx).K8sClient, !config.DisableServiceLB)
|
|
|
|
},
|
|
|
|
func(ctx context.Context) error {
|
|
|
|
dataDir := filepath.Join(controlConfig.DataDir, "manifests")
|
2019-01-22 21:14:58 +00:00
|
|
|
if err := deploy.Stage(dataDir); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2019-02-02 05:09:11 +00:00
|
|
|
if err := deploy.WatchFiles(ctx, controlConfig.Skips, dataDir); err != nil {
|
2019-01-22 21:14:58 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
},
|
|
|
|
},
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ctx, _, err = normanConfig.Build(ctx, nil)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
for {
|
|
|
|
certs, err := tlsServer.CACert()
|
|
|
|
if err != nil {
|
|
|
|
logrus.Infof("waiting to generate CA certs")
|
|
|
|
time.Sleep(time.Second)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
return certs, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func HomeKubeConfig() (string, error) {
|
|
|
|
return resolvehome.Resolve("${HOME}/.kube/k3s.yaml")
|
|
|
|
}
|
|
|
|
|
|
|
|
func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
|
|
|
|
var (
|
|
|
|
nodeFile string
|
|
|
|
)
|
|
|
|
|
|
|
|
if advertiseIP == "" {
|
|
|
|
advertiseIP = "localhost"
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(config.Runtime.NodeToken) > 0 {
|
|
|
|
p := filepath.Join(config.DataDir, "node-token")
|
|
|
|
if err := writeToken(config.Runtime.NodeToken, p, certs); err == nil {
|
|
|
|
logrus.Infof("Node token is available at %s", p)
|
|
|
|
nodeFile = p
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(nodeFile) > 0 {
|
2019-01-22 21:14:58 +00:00
|
|
|
printToken(tlsConfig.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
|
|
|
|
clientToken := FormatToken(config.Runtime.ClientToken, certs)
|
|
|
|
url := fmt.Sprintf("https://localhost:%d", tlsConfig.HTTPSPort)
|
|
|
|
kubeConfig, err := HomeKubeConfig()
|
|
|
|
def := true
|
|
|
|
if err != nil {
|
|
|
|
kubeConfig = filepath.Join(config.DataDir, "kubeconfig-k3s.yaml")
|
|
|
|
def = false
|
|
|
|
}
|
|
|
|
|
2019-01-22 21:14:58 +00:00
|
|
|
if config.KubeConfigOutput != "" {
|
|
|
|
kubeConfig = config.KubeConfigOutput
|
|
|
|
}
|
|
|
|
|
2019-01-01 08:23:01 +00:00
|
|
|
if err = clientaccess.AgentAccessInfoToKubeConfig(kubeConfig, url, clientToken); err != nil {
|
|
|
|
logrus.Errorf("Failed to generate kubeconfig: %v", err)
|
|
|
|
}
|
|
|
|
|
2019-01-22 21:14:58 +00:00
|
|
|
if config.KubeConfigMode != "" {
|
|
|
|
mode, err := strconv.ParseInt(config.KubeConfigMode, 8, 0)
|
|
|
|
if err == nil {
|
|
|
|
os.Chmod(kubeConfig, os.FileMode(mode))
|
|
|
|
} else {
|
2019-01-24 17:51:24 +00:00
|
|
|
logrus.Errorf("failed to set %s to mode %s: %v", kubeConfig, os.FileMode(mode), err)
|
2019-01-22 21:14:58 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-01-01 08:23:01 +00:00
|
|
|
logrus.Infof("Wrote kubeconfig %s", kubeConfig)
|
|
|
|
if def {
|
|
|
|
logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0]))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func setupDataDirAndChdir(config *config.Control) error {
|
|
|
|
var (
|
|
|
|
err error
|
|
|
|
)
|
|
|
|
|
|
|
|
config.DataDir, err = resolveDataDir(config.DataDir)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
dataDir := config.DataDir
|
|
|
|
|
|
|
|
if err := os.MkdirAll(dataDir, 0700); err != nil {
|
|
|
|
return errors.Wrapf(err, "can not mkdir %s", dataDir)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.Chdir(dataDir); err != nil {
|
|
|
|
return errors.Wrapf(err, "can not chdir %s", dataDir)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func readTokenFile(file string) (string, error) {
|
|
|
|
content, err := ioutil.ReadFile(file)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
return strings.TrimSpace(string(content)), nil
|
|
|
|
}
|
|
|
|
|
2019-01-22 21:14:58 +00:00
|
|
|
func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
|
2019-01-01 08:23:01 +00:00
|
|
|
ip := advertiseIP
|
|
|
|
if ip == "" {
|
|
|
|
hostIP, err := net.ChooseHostInterface()
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
ip = hostIP.String()
|
|
|
|
}
|
|
|
|
|
2019-01-22 21:14:58 +00:00
|
|
|
logrus.Infof("%s k3s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, cmd, ip, httpsPort)
|
2019-01-01 08:23:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func FormatToken(token string, certs string) string {
|
|
|
|
if len(token) == 0 {
|
|
|
|
return token
|
|
|
|
}
|
|
|
|
|
|
|
|
prefix := "K10"
|
|
|
|
if len(certs) > 0 {
|
|
|
|
digest := sha256.Sum256([]byte(certs))
|
|
|
|
prefix = "K10" + hex.EncodeToString(digest[:]) + "::"
|
|
|
|
}
|
|
|
|
|
|
|
|
return prefix + token
|
|
|
|
}
|
|
|
|
|
|
|
|
func writeToken(token, file, certs string) error {
|
|
|
|
if len(token) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
token = FormatToken(token, certs)
|
|
|
|
return ioutil.WriteFile(file, []byte(token+"\n"), 0600)
|
|
|
|
}
|