Fix storing bootstrap data with empty token string (#3422)

* Fix storing bootstrap data with empty token string

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* delete node password secret after restoration

fixes to bootstrap key

vendor update

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix comment

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix typo

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* more fixes

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fixes

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fixes

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* typos

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* Removing dynamic listener file after restoration

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* go mod tidy

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

go.mod

@@ -85,7 +85,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/k3s-io/helm-controller v0.9.1
-	github.com/k3s-io/kine v0.6.0
+	github.com/k3s-io/kine v0.6.1
 	github.com/klauspost/compress v1.12.2
 	github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
 	github.com/lib/pq v1.8.0

go.sum

@@ -542,8 +542,8 @@ github.com/k3s-io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea h1:7cwby0GoN
 github.com/k3s-io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
 github.com/k3s-io/helm-controller v0.9.1 h1:qtHWTNHiuCPRbA2YZ7z7jTgSHo7Yc5He52oMga72yUk=
 github.com/k3s-io/helm-controller v0.9.1/go.mod h1:nZP8FH3KZrNNUf5r+SwwiMR63HS6lxdHdpHijgPfF74=
-github.com/k3s-io/kine v0.6.0 h1:4l7wjgCxb2oD+7Hyf3xIhkGd/6s1sXpRFdQiyy+7Ki8=
-github.com/k3s-io/kine v0.6.0/go.mod h1:rzCs93+rQHZGOiewMd84PDrER92QeZ6eeHbWkfEy4+w=
+github.com/k3s-io/kine v0.6.1 h1:GcxxxtfmXIRiVqImZoGCHgJl6+S4Be3tY9nm2ArY7/U=
+github.com/k3s-io/kine v0.6.1/go.mod h1:rzCs93+rQHZGOiewMd84PDrER92QeZ6eeHbWkfEy4+w=
 github.com/k3s-io/kubernetes v1.21.1-k3s1 h1:X8nEv12/bI3iR2+ARLuzvosPW8iMOisMlklOAeovISw=
 github.com/k3s-io/kubernetes v1.21.1-k3s1/go.mod h1:ef++isEL1PW0taH6z7DXrSztPglrZ7jQhyvcMEtm0gQ=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.21.1-k3s1 h1:7iwn62FGlOqG9oRHwTY/+AbFlSZffWMqx6WUXjRpQPk=

pkg/cli/server/server.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/agent"
 	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/datadir"
 	"github.com/rancher/k3s/pkg/etcd"
 	"github.com/rancher/k3s/pkg/netutil"
@@ -395,7 +396,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	}
 
 	url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort)
-	token, err := server.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA)
+	token, err := clientaccess.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA)
 	if err != nil {
 		return err
 	}

pkg/clientaccess/token.go

@@ -287,3 +287,20 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro
 
 	return ioutil.ReadAll(resp.Body)
 }
+
+func FormatToken(token string, certFile string) (string, error) {
+	if len(token) == 0 {
+		return token, nil
+	}
+
+	certHash := ""
+	if len(certFile) > 0 {
+		bytes, err := ioutil.ReadFile(certFile)
+		if err != nil {
+			return "", nil
+		}
+		digest := sha256.Sum256(bytes)
+		certHash = tokenPrefix + hex.EncodeToString(digest[:]) + "::"
+	}
+	return certHash + token, nil
+}

pkg/cluster/bootstrap.go

@@ -132,7 +132,7 @@ func (c *Cluster) httpBootstrap() error {
 func (c *Cluster) bootstrap(ctx context.Context) error {
 	c.joining = true
 
-	// bootstrap managed database via HTTP
+	// bootstrap managed database via HTTPS
 	if c.runtime.HTTPBootstrap {
 		return c.httpBootstrap()
 	}

pkg/cluster/https.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"path/filepath"
 
 	"github.com/rancher/dynamiclistener"
@@ -14,6 +15,7 @@ import (
 	"github.com/rancher/dynamiclistener/storage/kubernetes"
 	"github.com/rancher/dynamiclistener/storage/memory"
 	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/etcd"
 	"github.com/rancher/k3s/pkg/version"
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
 	"github.com/sirupsen/logrus"
@@ -24,16 +26,21 @@ import (
 // dynamiclistener will use the cluster's Server CA to sign the dynamically generate certificate,
 // and will sync the certs into the Kubernetes datastore, with a local disk cache.
 func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, error) {
+	if c.managedDB != nil {
+		if _, err := os.Stat(etcd.ResetFile(c.config)); err == nil {
+			// delete the dynamic listener file if it exists after restoration to fix restoration
+			// on fresh nodes
+			os.Remove(filepath.Join(c.config.DataDir, "tls/dynamic-cert.json"))
+		}
+	}
 	tcp, err := dynamiclistener.NewTCPListener(c.config.BindAddress, c.config.SupervisorPort)
 	if err != nil {
 		return nil, nil, err
 	}
-
 	cert, key, err := factory.LoadCerts(c.runtime.ServerCA, c.runtime.ServerCAKey)
 	if err != nil {
 		return nil, nil, err
 	}
-
 	storage := tlsStorage(ctx, c.config.DataDir, c.runtime)
 	return dynamiclistener.NewListener(tcp, storage, cert, key, dynamiclistener.Config{
 		ExpirationDaysCheck: config.CertificateRenewDays,

pkg/cluster/managed.go

@@ -15,8 +15,10 @@ import (
 	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/rancher/k3s/pkg/cluster/managed"
 	"github.com/rancher/k3s/pkg/etcd"
+	"github.com/rancher/k3s/pkg/nodepassword"
 	"github.com/rancher/k3s/pkg/version"
 	"github.com/sirupsen/logrus"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 )
 
 // testClusterDB returns a channel that will be closed when the datastore connection is available.
@@ -76,6 +78,10 @@ func (c *Cluster) start(ctx context.Context) error {
 		return fmt.Errorf("cluster-reset was successfully performed, please remove the cluster-reset flag and start %s normally, if you need to perform another cluster reset, you must first manually delete the %s file", version.Program, resetFile)
 	}
 
+	if _, err := os.Stat(resetFile); err == nil {
+		// before removing reset file we need to delete the node passwd secret
+		go c.deleteNodePasswdSecret(ctx)
+	}
 	// removing the reset file and ignore error if the file doesn't exist
 	os.Remove(resetFile)
 
@@ -165,3 +171,31 @@ func (c *Cluster) setupEtcdProxy(ctx context.Context, etcdProxy etcd.Proxy) {
 		}
 	}()
 }
+
+// deleteNodePasswdSecret wipes out the node password secret after restoration
+func (c *Cluster) deleteNodePasswdSecret(ctx context.Context) {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	for range t.C {
+		nodeName := os.Getenv("NODE_NAME")
+		if nodeName == "" {
+			logrus.Infof("waiting for node name to be set")
+			continue
+		}
+		if c.runtime == nil {
+			logrus.Infof("runtime is not yet initialized")
+			continue
+		}
+		secretsClient := c.runtime.Core.Core().V1().Secret()
+		if err := nodepassword.Delete(secretsClient, nodeName); err != nil {
+			if apierrors.IsNotFound(err) {
+				logrus.Debugf("node password secret is not found for node %s", nodeName)
+				return
+			}
+			logrus.Warnf("failed to delete old node password secret: %v", err)
+			continue
+		}
+		return
+	}
+
+}

pkg/cluster/storage.go

@@ -3,10 +3,15 @@ package cluster
 import (
 	"bytes"
 	"context"
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/k3s-io/kine/pkg/client"
 	"github.com/rancher/k3s/pkg/bootstrap"
+	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/sirupsen/logrus"
 )
 
@@ -19,8 +24,20 @@ func (c *Cluster) save(ctx context.Context) error {
 	if err := bootstrap.Write(buf, &c.runtime.ControlRuntimeBootstrap); err != nil {
 		return err
 	}
+	token := c.config.Token
+	if token == "" {
+		tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir)
+		if err != nil {
+			return err
+		}
+		token = tokenFromFile
+	}
+	normalizedToken, err := normalizeToken(token)
+	if err != nil {
+		return err
+	}
 
-	data, err := encrypt(c.config.Token, buf.Bytes())
+	data, err := encrypt(normalizedToken, buf.Bytes())
 	if err != nil {
 		return err
 	}
@@ -30,12 +47,17 @@ func (c *Cluster) save(ctx context.Context) error {
 		return err
 	}
 
-	if err := storageClient.Create(ctx, storageKey(c.config.Token), data); err != nil {
+	_, _, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken)
+	if err != nil {
+		return err
+	}
+
+	if err := storageClient.Create(ctx, storageKey(normalizedToken), data); err != nil {
 		if err.Error() == "key exists" {
-			logrus.Warnln("Bootstrap key exists. Please follow documentation updating a node after restore.")
+			logrus.Warnln("bootstrap key exists; please follow documentation on updating a node after snapshot restore")
 			return nil
 		} else if strings.Contains(err.Error(), "not supported for learner") {
-			logrus.Debug("Skipping bootstrap data save on learner.")
+			logrus.Debug("skipping bootstrap data save on learner")
 			return nil
 		}
 		return err
@@ -46,6 +68,7 @@ func (c *Cluster) save(ctx context.Context) error {
 
 // storageBootstrap loads data from the datastore into the ControlRuntimeBootstrap struct.
 // The storage key and encryption passphrase are both derived from the join token.
+// token is either passed
 func (c *Cluster) storageBootstrap(ctx context.Context) error {
 	if err := c.startStorage(ctx); err != nil {
 		return err
@@ -56,18 +79,104 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 		return err
 	}
 
-	value, err := storageClient.Get(ctx, storageKey(c.config.Token))
-	if err == client.ErrNotFound {
-		c.saveBootstrap = true
-		return nil
-	} else if err != nil {
+	token := c.config.Token
+	if token == "" {
+		tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir)
+		if err != nil {
+			return err
+		}
+		if tokenFromFile == "" {
+			// at this point this is a fresh start in a non managed environment
+			c.saveBootstrap = true
+			return nil
+		}
+		token = tokenFromFile
+	}
+	normalizedToken, err := normalizeToken(token)
+	if err != nil {
 		return err
 	}
 
-	data, err := decrypt(c.config.Token, value.Data)
+	value, emptyKey, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken)
+	if err != nil {
+		return err
+	}
+	if value == nil {
+		return nil
+	}
+	if emptyKey {
+		normalizedToken = ""
+	}
+	data, err := decrypt(normalizedToken, value.Data)
 	if err != nil {
 		return err
 	}
 
 	return bootstrap.Read(bytes.NewBuffer(data), &c.runtime.ControlRuntimeBootstrap)
 }
+
+// getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is
+// hashed with empty string and will check for any key that is hashed by different token than the one
+// passed to it, it will return error if it finds a key that is hashed with different token and will return
+// value if it finds the key hashed by passed token or empty string
+func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, token string) (*client.Value, bool, error) {
+	emptyStringKey := storageKey("")
+	tokenKey := storageKey(token)
+
+	bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0)
+	if err != nil {
+		return nil, false, err
+	}
+	if len(bootstrapList) == 0 {
+		c.saveBootstrap = true
+		return nil, false, nil
+	}
+	if len(bootstrapList) > 1 {
+		return nil, false, errors.New("found more than one bootstrap keys in storage")
+	}
+	bootstrapKV := bootstrapList[0]
+	// checking for empty string bootstrap key
+	switch string(bootstrapKV.Key) {
+	case emptyStringKey:
+		logrus.Warn("bootstrap data already found and encrypted with empty string, deleting empty key")
+		c.saveBootstrap = true
+		if err := storageClient.Delete(ctx, emptyStringKey); err != nil {
+			return nil, false, err
+		}
+		return &bootstrapKV, true, nil
+	case tokenKey:
+		return &bootstrapKV, false, nil
+	}
+
+	return nil, false, errors.New("bootstrap data already found and encrypted with different token")
+}
+
+// readTokenFromFile will attempt to get the token from <data-dir>/token if it the file not found
+// in case of fresh installation it will try to use the runtime serverToken saved in memory
+// after stripping it from any additional information like the username or cahash, if the file
+// found then it will still strip the token from any additional info
+func readTokenFromFile(serverToken, certs, dataDir string) (string, error) {
+	tokenFile := filepath.Join(dataDir, "token")
+	b, err := ioutil.ReadFile(tokenFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			token, err := clientaccess.FormatToken(serverToken, certs)
+			if err != nil {
+				return token, err
+			}
+			return token, nil
+		}
+		return "", err
+	}
+	// strip the token from any new line if its read from file
+	return string(bytes.TrimRight(b, "\n")), nil
+}
+
+// normalizeToken will normalize the token read from file or passed as a cli flag
+func normalizeToken(token string) (string, error) {
+	_, password, ok := clientaccess.ParseUsernamePassword(token)
+	if !ok {
+		return password, errors.New("failed to normalize token")
+	}
+	return password, nil
+}

pkg/server/server.go

@@ -2,8 +2,6 @@ package server
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"fmt"
 	"io/ioutil"
 	net2 "net"
@@ -423,30 +421,12 @@ func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
 	logrus.Infof("%s %s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, version.Program, cmd, ip, httpsPort)
 }
 
-func FormatToken(token string, certFile string) (string, error) {
-	if len(token) == 0 {
-		return token, nil
-	}
-
-	prefix := "K10"
-	if len(certFile) > 0 {
-		bytes, err := ioutil.ReadFile(certFile)
-		if err != nil {
-			return "", nil
-		}
-		digest := sha256.Sum256(bytes)
-		prefix = "K10" + hex.EncodeToString(digest[:]) + "::"
-	}
-
-	return prefix + token, nil
-}
-
 func writeToken(token, file, certs string) error {
 	if len(token) == 0 {
 		return nil
 	}
 
-	token, err := FormatToken(token, certs)
+	token, err := clientaccess.FormatToken(token, certs)
 	if err != nil {
 		return err
 	}

vendor/github.com/k3s-io/kine/pkg/client/client.go

@@ -26,6 +26,7 @@ type Client interface {
 	Put(ctx context.Context, key string, value []byte) error
 	Create(ctx context.Context, key string, value []byte) error
 	Update(ctx context.Context, key string, revision int64, value []byte) error
+	Delete(ctx context.Context, key string) error
 	Close() error
 }
 
@@ -128,6 +129,11 @@ func (c *client) Update(ctx context.Context, key string, revision int64, value [
 	return nil
 }
 
+func (c *client) Delete(ctx context.Context, key string) error {
+	_, err := c.c.Delete(ctx, key)
+	return err
+}
+
 func (c *client) Close() error {
 	return c.c.Close()
 }

vendor/modules.txt

@@ -707,7 +707,7 @@ github.com/k3s-io/helm-controller/pkg/generated/informers/externalversions/helm.
 github.com/k3s-io/helm-controller/pkg/generated/informers/externalversions/internalinterfaces
 github.com/k3s-io/helm-controller/pkg/generated/listers/helm.cattle.io/v1
 github.com/k3s-io/helm-controller/pkg/helm
-# github.com/k3s-io/kine v0.6.0
+# github.com/k3s-io/kine v0.6.1
 ## explicit
 github.com/k3s-io/kine/pkg/broadcaster
 github.com/k3s-io/kine/pkg/client