package cluster import ( "bytes" "context" "errors" "io/ioutil" "os" "path/filepath" "strings" "github.com/k3s-io/kine/pkg/client" "github.com/rancher/k3s/pkg/bootstrap" "github.com/rancher/k3s/pkg/clientaccess" "github.com/sirupsen/logrus" ) // save writes the current ControlRuntimeBootstrap data to the datastore. This contains a complete // snapshot of the cluster's CA certs and keys, encryption passphrases, etc - encrypted with the join token. // This is used when bootstrapping a cluster from a managed database or external etcd cluster. // This is NOT used with embedded etcd, which bootstraps over HTTP. func (c *Cluster) save(ctx context.Context) error { buf := &bytes.Buffer{} if err := bootstrap.Write(buf, &c.runtime.ControlRuntimeBootstrap); err != nil { return err } token := c.config.Token if token == "" { tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir) if err != nil { return err } token = tokenFromFile } normalizedToken, err := normalizeToken(token) if err != nil { return err } data, err := encrypt(normalizedToken, buf.Bytes()) if err != nil { return err } storageClient, err := client.New(c.etcdConfig) if err != nil { return err } _, _, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) if err != nil { return err } if err := storageClient.Create(ctx, storageKey(normalizedToken), data); err != nil { if err.Error() == "key exists" { logrus.Warnln("bootstrap key exists; please follow documentation on updating a node after snapshot restore") return nil } else if strings.Contains(err.Error(), "not supported for learner") { logrus.Debug("skipping bootstrap data save on learner") return nil } return err } return nil } // storageBootstrap loads data from the datastore into the ControlRuntimeBootstrap struct. // The storage key and encryption passphrase are both derived from the join token. // token is either passed func (c *Cluster) storageBootstrap(ctx context.Context) error { if err := c.startStorage(ctx); err != nil { return err } storageClient, err := client.New(c.etcdConfig) if err != nil { return err } token := c.config.Token if token == "" { tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir) if err != nil { return err } if tokenFromFile == "" { // at this point this is a fresh start in a non managed environment c.saveBootstrap = true return nil } token = tokenFromFile } normalizedToken, err := normalizeToken(token) if err != nil { return err } value, emptyKey, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) if err != nil { return err } if value == nil { return nil } if emptyKey { normalizedToken = "" } data, err := decrypt(normalizedToken, value.Data) if err != nil { return err } return bootstrap.Read(bytes.NewBuffer(data), &c.runtime.ControlRuntimeBootstrap) } // getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is // hashed with empty string and will check for any key that is hashed by different token than the one // passed to it, it will return error if it finds a key that is hashed with different token and will return // value if it finds the key hashed by passed token or empty string func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, token string) (*client.Value, bool, error) { emptyStringKey := storageKey("") tokenKey := storageKey(token) bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0) if err != nil { return nil, false, err } if len(bootstrapList) == 0 { c.saveBootstrap = true return nil, false, nil } if len(bootstrapList) > 1 { logrus.Warn("found multiple bootstrap keys in storage") } for _, bootstrapKV := range bootstrapList { // checking for empty string bootstrap key switch string(bootstrapKV.Key) { case emptyStringKey: logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token") c.saveBootstrap = true if err := storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified); err != nil { return nil, false, err } return &bootstrapKV, true, nil case tokenKey: return &bootstrapKV, false, nil } } return nil, false, errors.New("bootstrap data already found and encrypted with different token") } // readTokenFromFile will attempt to get the token from /token if it the file not found // in case of fresh installation it will try to use the runtime serverToken saved in memory // after stripping it from any additional information like the username or cahash, if the file // found then it will still strip the token from any additional info func readTokenFromFile(serverToken, certs, dataDir string) (string, error) { tokenFile := filepath.Join(dataDir, "token") b, err := ioutil.ReadFile(tokenFile) if err != nil { if os.IsNotExist(err) { token, err := clientaccess.FormatToken(serverToken, certs) if err != nil { return token, err } return token, nil } return "", err } // strip the token from any new line if its read from file return string(bytes.TrimRight(b, "\n")), nil } // normalizeToken will normalize the token read from file or passed as a cli flag func normalizeToken(token string) (string, error) { _, password, ok := clientaccess.ParseUsernamePassword(token) if !ok { return password, errors.New("failed to normalize token") } return password, nil }