diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 6826c9b007..52bb444d71 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -23,7 +23,7 @@ type Server struct { ExtraSchedulerArgs cli.StringSlice ExtraControllerArgs cli.StringSlice Rootless bool - CertStorageBackend string + BootstrapType string StorageBackend string StorageEndpoint string StorageCAFile string @@ -146,9 +146,9 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.Rootless, }, cli.StringFlag{ - Name: "cert-storage-backend", - Usage: "(experimental) Specify storage type for storing certificate information", - Destination: &ServerConfig.CertStorageBackend, + Name: "bootstrap", + Usage: "(experimental) Specify data bootstrap behavior (one of: none, read, write, or full), etcd3 only", + Destination: &ServerConfig.BootstrapType, }, cli.StringFlag{ Name: "storage-backend", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 297ed73cee..05d06fed94 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -124,7 +124,7 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort - serverConfig.ControlConfig.CertStorageBackend = cfg.CertStorageBackend + serverConfig.ControlConfig.BootstrapType = cfg.BootstrapType if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" { serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index ae7f39de02..7a0d4104ef 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -81,7 +81,7 @@ type Control struct { KubeConfigMode string DataDir string Skips []string - CertStorageBackend string + BootstrapType string StorageBackend string StorageEndpoint string StorageCAFile string diff --git a/pkg/daemons/control/bootstrap.go b/pkg/daemons/control/bootstrap.go new file mode 100644 index 0000000000..b5ab233942 --- /dev/null +++ b/pkg/daemons/control/bootstrap.go @@ -0,0 +1,237 @@ +package control + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "strings" + "time" + + "encoding/base64" + + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/sirupsen/logrus" + "go.etcd.io/etcd/clientv3" +) + +const ( + etcdDialTimeout = 5 * time.Second + k3sRuntimeEtcdPath = "/k3s/runtime" + bootstrapTypeNone = "none" + bootstrapTypeRead = "read" + bootstrapTypeWrite = "write" + bootstrapTypeFull = "full" +) + +type serverBootstrap struct { + ServerCAData string `json:"serverCAData,omitempty"` + ServerCAKeyData string `json:"serverCAKeyData,omitempty"` + ClientCAData string `json:"clientCAData,omitempty"` + ClientCAKeyData string `json:"clientCAKeyData,omitempty"` + ServiceKeyData string `json:"serviceKeyData,omitempty"` + PasswdFileData string `json:"passwdFileData,omitempty"` + RequestHeaderCAData string `json:"requestHeaderCAData,omitempty"` + RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"` +} + +var validBootstrapTypes = map[string]bool{ + bootstrapTypeRead: true, + bootstrapTypeWrite: true, + bootstrapTypeFull: true, +} + +func fetchBootstrapData(cfg *config.Control) error { + if valid, err := checkBootstrapArgs(cfg, map[string]bool{ + bootstrapTypeFull: true, + bootstrapTypeRead: true, + }); !valid { + if err != nil { + logrus.Warnf("Not fetching bootstrap data: %v", err) + } + return nil + } + + tlsConfig, err := genBootstrapTLSConfig(cfg) + if err != nil { + return err + } + + endpoints := strings.Split(cfg.StorageEndpoint, ",") + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: etcdDialTimeout, + TLS: tlsConfig, + }) + if err != nil { + return err + } + defer cli.Close() + + gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) + if err != nil { + return err + } + if len(gr.Kvs) == 0 { + return nil + } + + runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value)) + if err != nil { + return err + } + serverRuntime := &serverBootstrap{} + if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil { + return err + } + return writeRuntimeBootstrapData(cfg.Runtime, serverRuntime) +} + +func storeBootstrapData(cfg *config.Control) error { + if valid, err := checkBootstrapArgs(cfg, map[string]bool{ + bootstrapTypeFull: true, + bootstrapTypeWrite: true, + }); !valid { + if err != nil { + logrus.Warnf("Not storing boostrap data: %v", err) + } + return nil + } + + tlsConfig, err := genBootstrapTLSConfig(cfg) + if err != nil { + return err + } + + endpoints := strings.Split(cfg.StorageEndpoint, ",") + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: etcdDialTimeout, + TLS: tlsConfig, + }) + if err != nil { + return err + } + defer cli.Close() + + gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) + if err != nil { + return err + } + if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" { + return nil + } + certData, err := readRuntimeBootstrapData(cfg.Runtime) + if err != nil { + return err + } + + runtimeBase64 := base64.StdEncoding.EncodeToString(certData) + _, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64) + if err != nil { + return err + } + + return nil +} + +func checkBootstrapArgs(cfg *config.Control, accepted map[string]bool) (bool, error) { + if cfg.BootstrapType == "" || cfg.BootstrapType == bootstrapTypeNone { + return false, nil + } + if !validBootstrapTypes[cfg.BootstrapType] { + return false, fmt.Errorf("unsupported bootstrap type [%s]", cfg.BootstrapType) + } + if cfg.StorageBackend != "etcd3" { + return false, errors.New("bootstrap only supported with etcd3 as storage backend") + } + if !accepted[cfg.BootstrapType] { + return false, nil + } + return true, nil +} + +func genBootstrapTLSConfig(cfg *config.Control) (*tls.Config, error) { + tlsConfig := &tls.Config{} + if cfg.StorageCertFile != "" && cfg.StorageKeyFile != "" { + certPem, err := ioutil.ReadFile(cfg.StorageCertFile) + if err != nil { + return nil, err + } + keyPem, err := ioutil.ReadFile(cfg.StorageKeyFile) + if err != nil { + return nil, err + } + tlsCert, err := tls.X509KeyPair(certPem, keyPem) + if err != nil { + return nil, err + } + tlsConfig.Certificates = []tls.Certificate{tlsCert} + } + if cfg.StorageCAFile != "" { + caData, err := ioutil.ReadFile(cfg.StorageCAFile) + if err != nil { + return nil, err + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(caData) + tlsConfig.RootCAs = certPool + } + return tlsConfig, nil +} + +func readRuntimeBootstrapData(runtime *config.ControlRuntime) ([]byte, error) { + serverBootstrapFiles := map[string]string{ + runtime.ServerCA: "", + runtime.ServerCAKey: "", + runtime.ClientCA: "", + runtime.ClientCAKey: "", + runtime.ServiceKey: "", + runtime.PasswdFile: "", + runtime.RequestHeaderCA: "", + runtime.RequestHeaderCAKey: "", + } + for k := range serverBootstrapFiles { + data, err := ioutil.ReadFile(k) + if err != nil { + return nil, err + } + serverBootstrapFiles[k] = string(data) + } + serverBootstrapFileData := &serverBootstrap{ + ServerCAData: serverBootstrapFiles[runtime.ServerCA], + ServerCAKeyData: serverBootstrapFiles[runtime.ServerCAKey], + ClientCAData: serverBootstrapFiles[runtime.ClientCA], + ClientCAKeyData: serverBootstrapFiles[runtime.ClientCAKey], + ServiceKeyData: serverBootstrapFiles[runtime.ServiceKey], + PasswdFileData: serverBootstrapFiles[runtime.PasswdFile], + RequestHeaderCAData: serverBootstrapFiles[runtime.RequestHeaderCA], + RequestHeaderCAKeyData: serverBootstrapFiles[runtime.RequestHeaderCAKey], + } + return json.Marshal(serverBootstrapFileData) +} + +func writeRuntimeBootstrapData(runtime *config.ControlRuntime, runtimeData *serverBootstrap) error { + runtimePathValue := map[string]string{ + runtime.ServerCA: runtimeData.ServerCAData, + runtime.ServerCAKey: runtimeData.ServerCAKeyData, + runtime.ClientCA: runtimeData.ClientCAData, + runtime.ClientCAKey: runtimeData.ClientCAKeyData, + runtime.ServiceKey: runtimeData.ServiceKeyData, + runtime.PasswdFile: runtimeData.PasswdFileData, + runtime.RequestHeaderCA: runtimeData.RequestHeaderCAData, + runtime.RequestHeaderCAKey: runtimeData.RequestHeaderCAKeyData, + } + for k, v := range runtimePathValue { + if _, err := os.Stat(k); os.IsNotExist(err) { + if err := ioutil.WriteFile(k, []byte(v), 600); err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/daemons/control/ha.go b/pkg/daemons/control/ha.go deleted file mode 100644 index 6f1784c616..0000000000 --- a/pkg/daemons/control/ha.go +++ /dev/null @@ -1,152 +0,0 @@ -package control - -import ( - "context" - "encoding/json" - "io/ioutil" - "os" - "strings" - "time" - - "encoding/base64" - - "github.com/rancher/k3s/pkg/daemons/config" - "go.etcd.io/etcd/clientv3" -) - -const ( - etcdDialTimeout = 5 * time.Second - k3sRuntimeEtcdPath = "/k3s/runtime" -) - -type serverHA struct { - ServerCAData string `json:"serverCAData,omitempty"` - ServerCAKeyData string `json:"serverCAKeyData,omitempty"` - ClientCAData string `json:"clientCAData,omitempty"` - ClientCAKeyData string `json:"clientCAKeyData,omitempty"` - ServiceKeyData string `json:"serviceKeyData,omitempty"` - PasswdFileData string `json:"passwdFileData,omitempty"` - RequestHeaderCAData string `json:"requestHeaderCAData,omitempty"` - RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"` -} - -func setHAData(cfg *config.Control) error { - if cfg.StorageBackend != "etcd3" || cfg.CertStorageBackend != "etcd3" { - return nil - } - - endpoints := strings.Split(cfg.StorageEndpoint, ",") - cli, err := clientv3.New(clientv3.Config{ - Endpoints: endpoints, - DialTimeout: etcdDialTimeout, - }) - if err != nil { - return err - } - defer cli.Close() - - gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) - if err != nil { - return err - } - if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" { - return nil - } - certData, err := readRuntimeCertData(cfg.Runtime) - if err != nil { - return err - } - - runtimeBase64 := base64.StdEncoding.EncodeToString(certData) - _, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64) - if err != nil { - return err - } - - return nil -} - -func getHAData(cfg *config.Control) error { - serverRuntime := &serverHA{} - if cfg.StorageBackend != "etcd3" || cfg.CertStorageBackend != "etcd3" { - return nil - } - endpoints := strings.Split(cfg.StorageEndpoint, ",") - cli, err := clientv3.New(clientv3.Config{ - Endpoints: endpoints, - DialTimeout: etcdDialTimeout, - }) - if err != nil { - return err - } - defer cli.Close() - - gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) - if err != nil { - return err - } - if len(gr.Kvs) == 0 { - return nil - } - - runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value)) - if err != nil { - return err - } - if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil { - return err - } - return writeRuntimeCertData(cfg.Runtime, serverRuntime) -} - -func readRuntimeCertData(runtime *config.ControlRuntime) ([]byte, error) { - serverHACerts := map[string]string{ - runtime.ServerCA: "", - runtime.ServerCAKey: "", - runtime.ClientCA: "", - runtime.ClientCAKey: "", - runtime.ServiceKey: "", - runtime.PasswdFile: "", - runtime.RequestHeaderCA: "", - runtime.RequestHeaderCAKey: "", - } - for k := range serverHACerts { - data, err := ioutil.ReadFile(k) - if err != nil { - return nil, err - } - serverHACerts[k] = string(data) - } - serverHACertsData := &serverHA{ - ServerCAData: serverHACerts[runtime.ServerCA], - ServerCAKeyData: serverHACerts[runtime.ServerCAKey], - ClientCAData: serverHACerts[runtime.ClientCA], - ClientCAKeyData: serverHACerts[runtime.ClientCAKey], - ServiceKeyData: serverHACerts[runtime.ServiceKey], - PasswdFileData: serverHACerts[runtime.PasswdFile], - RequestHeaderCAData: serverHACerts[runtime.RequestHeaderCA], - RequestHeaderCAKeyData: serverHACerts[runtime.RequestHeaderCAKey], - } - return json.Marshal(serverHACertsData) -} - -func writeRuntimeCertData(runtime *config.ControlRuntime, runtimeData *serverHA) error { - runtimePathValue := map[string]string{ - runtime.ServerCA: runtimeData.ServerCAData, - runtime.ServerCAKey: runtimeData.ServerCAKeyData, - runtime.ClientCA: runtimeData.ClientCAData, - runtime.ClientCAKey: runtimeData.ClientCAKeyData, - runtime.ServiceKey: runtimeData.ServiceKeyData, - runtime.PasswdFile: runtimeData.PasswdFileData, - runtime.RequestHeaderCA: runtimeData.RequestHeaderCAData, - runtime.RequestHeaderCAKey: runtimeData.RequestHeaderCAKeyData, - } - for k, v := range runtimePathValue { - if _, err := os.Stat(k); os.IsNotExist(err) { - if err := ioutil.WriteFile(k, []byte(v), 600); err != nil { - return err - } - } - } - return nil -} diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 7b0c8be358..2c9ffdb78a 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -290,7 +290,7 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt") runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") - if err := getHAData(config); err != nil { + if err := fetchBootstrapData(config); err != nil { return err } @@ -306,7 +306,7 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { return err } - if err := setHAData(config); err != nil { + if err := storeBootstrapData(config); err != nil { return err }