From a65e5b6466cbbf2643cde059ef5d6142d73adfdc Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 21 Jul 2021 02:50:36 +0200 Subject: [PATCH 1/8] Fix multiple bootstrap keys found Signed-off-by: galal-hussein --- pkg/cluster/storage.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index e16bf6a8b5..ca31ae9464 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -132,20 +132,21 @@ func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient return nil, false, nil } if len(bootstrapList) > 1 { - return nil, false, errors.New("found multiple bootstrap keys in storage") + logrus.Warn("found multiple bootstrap keys in storage") } - bootstrapKV := bootstrapList[0] - // checking for empty string bootstrap key - switch string(bootstrapKV.Key) { - case emptyStringKey: - logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token") - c.saveBootstrap = true - if err := storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified); err != nil { - return nil, false, err + for _, bootstrapKV := range bootstrapList { + // checking for empty string bootstrap key + switch string(bootstrapKV.Key) { + case emptyStringKey: + logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token") + c.saveBootstrap = true + if err := storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified); err != nil { + return nil, false, err + } + return &bootstrapKV, true, nil + case tokenKey: + return &bootstrapKV, false, nil } - return &bootstrapKV, true, nil - case tokenKey: - return &bootstrapKV, false, nil } return nil, false, errors.New("bootstrap data already found and encrypted with different token") From ad17292fa85ebe62b0e2be3a9f81e6d0538f09af Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 21 Jul 2021 19:21:35 +0200 Subject: [PATCH 2/8] migrate empty string key properly Signed-off-by: galal-hussein --- pkg/cluster/storage.go | 76 +++++++++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index ca31ae9464..1b6031a8ed 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -47,7 +47,7 @@ func (c *Cluster) save(ctx context.Context) error { return err } - _, _, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) + _, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) if err != nil { return err } @@ -97,16 +97,14 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { return err } - value, emptyKey, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) + value, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) if err != nil { return err } if value == nil { return nil } - if emptyKey { - normalizedToken = "" - } + data, err := decrypt(normalizedToken, value.Data) if err != nil { return err @@ -119,37 +117,39 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { // hashed with empty string and will check for any key that is hashed by different token than the one // passed to it, it will return error if it finds a key that is hashed with different token and will return // value if it finds the key hashed by passed token or empty string -func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, token string) (*client.Value, bool, error) { +func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken string) (*client.Value, error) { emptyStringKey := storageKey("") - tokenKey := storageKey(token) - + tokenKey := storageKey(normalizedToken) bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0) if err != nil { - return nil, false, err + return nil, err } if len(bootstrapList) == 0 { c.saveBootstrap = true - return nil, false, nil + return nil, nil } if len(bootstrapList) > 1 { logrus.Warn("found multiple bootstrap keys in storage") } + // check for empty string key + if err := c.migrateEmptyStringToken(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken); err != nil { + return nil, err + } + + // getting the list of bootstrap again after migrating the empty key + bootstrapList, err = storageClient.List(ctx, "/bootstrap", 0) + if err != nil { + return nil, err + } for _, bootstrapKV := range bootstrapList { // checking for empty string bootstrap key switch string(bootstrapKV.Key) { - case emptyStringKey: - logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token") - c.saveBootstrap = true - if err := storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified); err != nil { - return nil, false, err - } - return &bootstrapKV, true, nil case tokenKey: - return &bootstrapKV, false, nil + return &bootstrapKV, nil } } - return nil, false, errors.New("bootstrap data already found and encrypted with different token") + return nil, errors.New("bootstrap data already found and encrypted with different token") } // readTokenFromFile will attempt to get the token from /token if it the file not found @@ -181,3 +181,41 @@ func normalizeToken(token string) (string, error) { } return password, nil } + +// migrateEmptyStringToken will list all keys that has prefix /bootstrap and will check for key that is +// hashed with empty string and then migrate this empty string key and resave it with the right token +func (c *Cluster) migrateEmptyStringToken(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token string) error { + for _, bootstrapKV := range bootstrapList { + // checking for empty string bootstrap key + if string(bootstrapKV.Key) == emptyStringKey { + logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token") + // making sure that the process is autonmous by decrypting and rencrypting the data first + data, err := decrypt("", bootstrapKV.Data) + if err != nil { + return err + } + encryptedData, err := encrypt(token, data) + if err != nil { + return err + } + // saving the new encrypted data with the right token key + if err := storageClient.Create(ctx, tokenKey, encryptedData); err != nil { + if err.Error() == "key exists" { + logrus.Warnln("bootstrap key exists; please follow documentation on updating a node after snapshot restore") + return nil + } else if strings.Contains(err.Error(), "not supported for learner") { + logrus.Debug("skipping bootstrap data save on learner") + return nil + } + return err + } + // deleting the empty string key + if err := storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified); err != nil { + return err + } + return nil + } + } + + return nil +} From 997ed7b9b46e39c602bd71a56df3b1ee80cef552 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 21 Jul 2021 19:56:12 +0200 Subject: [PATCH 3/8] simplifying the code Signed-off-by: galal-hussein --- pkg/cluster/storage.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 1b6031a8ed..2751515d43 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -143,8 +143,7 @@ func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient } for _, bootstrapKV := range bootstrapList { // checking for empty string bootstrap key - switch string(bootstrapKV.Key) { - case tokenKey: + if string(bootstrapKV.Key) == tokenKey { return &bootstrapKV, nil } } @@ -189,7 +188,7 @@ func (c *Cluster) migrateEmptyStringToken(ctx context.Context, bootstrapList []c // checking for empty string bootstrap key if string(bootstrapKV.Key) == emptyStringKey { logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token") - // making sure that the process is autonmous by decrypting and rencrypting the data first + // make sure that the process is non-destructive by decrypting/re-encrypting/storing the data before deleting the old key data, err := decrypt("", bootstrapKV.Data) if err != nil { return err @@ -210,10 +209,7 @@ func (c *Cluster) migrateEmptyStringToken(ctx context.Context, bootstrapList []c return err } // deleting the empty string key - if err := storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified); err != nil { - return err - } - return nil + return storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified) } } From b37783914843c23e0882bff808f1bee4041e0d13 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 21 Jul 2021 20:59:57 +0200 Subject: [PATCH 4/8] migrate old token key format Signed-off-by: galal-hussein --- pkg/cluster/storage.go | 67 +++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 2751515d43..4713fec5ca 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -47,7 +47,7 @@ func (c *Cluster) save(ctx context.Context) error { return err } - _, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) + _, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) if err != nil { return err } @@ -97,7 +97,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { return err } - value, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) + value, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token) if err != nil { return err } @@ -117,7 +117,7 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { // hashed with empty string and will check for any key that is hashed by different token than the one // passed to it, it will return error if it finds a key that is hashed with different token and will return // value if it finds the key hashed by passed token or empty string -func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken string) (*client.Value, error) { +func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, normalizedToken, oldToken string) (*client.Value, error) { emptyStringKey := storageKey("") tokenKey := storageKey(normalizedToken) bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0) @@ -131,8 +131,8 @@ func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient if len(bootstrapList) > 1 { logrus.Warn("found multiple bootstrap keys in storage") } - // check for empty string key - if err := c.migrateEmptyStringToken(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken); err != nil { + // check for empty string key and for old token format with k10 prefix + if err := c.migrateOldTokens(ctx, bootstrapList, storageClient, emptyStringKey, tokenKey, normalizedToken, oldToken); err != nil { return nil, err } @@ -181,37 +181,50 @@ func normalizeToken(token string) (string, error) { return password, nil } -// migrateEmptyStringToken will list all keys that has prefix /bootstrap and will check for key that is -// hashed with empty string and then migrate this empty string key and resave it with the right token -func (c *Cluster) migrateEmptyStringToken(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token string) error { +// migrateOldTokens will list all keys that has prefix /bootstrap and will check for key that is +// hashed with empty string and keys that is hashed with old token format before normalizing +// then migrate those and resave only with the normalized token +func (c *Cluster) migrateOldTokens(ctx context.Context, bootstrapList []client.Value, storageClient client.Client, emptyStringKey, tokenKey, token, oldToken string) error { + oldTokenKey := storageKey(oldToken) for _, bootstrapKV := range bootstrapList { // checking for empty string bootstrap key if string(bootstrapKV.Key) == emptyStringKey { logrus.Warn("bootstrap data encrypted with empty string, deleting and resaving with token") - // make sure that the process is non-destructive by decrypting/re-encrypting/storing the data before deleting the old key - data, err := decrypt("", bootstrapKV.Data) - if err != nil { + if err := doMigrateToken(ctx, storageClient, bootstrapKV, "", emptyStringKey, token, tokenKey); err != nil { return err } - encryptedData, err := encrypt(token, data) - if err != nil { + } else if string(bootstrapKV.Key) == oldTokenKey && oldTokenKey != tokenKey { + logrus.Warn("bootstrap data encrypted with old token format string, deleting and resaving with token") + if err := doMigrateToken(ctx, storageClient, bootstrapKV, oldToken, oldTokenKey, token, tokenKey); err != nil { return err } - // saving the new encrypted data with the right token key - if err := storageClient.Create(ctx, tokenKey, encryptedData); err != nil { - if err.Error() == "key exists" { - logrus.Warnln("bootstrap key exists; please follow documentation on updating a node after snapshot restore") - return nil - } else if strings.Contains(err.Error(), "not supported for learner") { - logrus.Debug("skipping bootstrap data save on learner") - return nil - } - return err - } - // deleting the empty string key - return storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified) } } - return nil } + +func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue client.Value, oldToken, oldTokenKey, newToken, newTokenKey string) error { + // make sure that the process is non-destructive by decrypting/re-encrypting/storing the data before deleting the old key + data, err := decrypt(oldToken, keyValue.Data) + if err != nil { + return err + } + encryptedData, err := encrypt(newToken, data) + if err != nil { + return err + } + // saving the new encrypted data with the right token key + if err := storageClient.Create(ctx, newTokenKey, encryptedData); err != nil { + if err.Error() == "key exists" { + logrus.Warnln("bootstrap key exists") + } else if strings.Contains(err.Error(), "not supported for learner") { + logrus.Debug("skipping bootstrap data save on learner") + return nil + } else { + return err + } + } + logrus.Infof("created bootstrap key %s", newTokenKey) + // deleting the old key + return storageClient.Delete(ctx, oldTokenKey, keyValue.Modified) +} From 2f82bfcf67b2c41b4496a5600896d6a13fc61a3c Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 21 Jul 2021 22:05:43 +0200 Subject: [PATCH 5/8] fix warning msg Signed-off-by: galal-hussein --- pkg/cluster/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 4713fec5ca..087a00bc8e 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -54,7 +54,7 @@ func (c *Cluster) save(ctx context.Context) error { if err := storageClient.Create(ctx, storageKey(normalizedToken), data); err != nil { if err.Error() == "key exists" { - logrus.Warnln("bootstrap key exists; please follow documentation on updating a node after snapshot restore") + logrus.Warnln("bootstrap key already exists") return nil } else if strings.Contains(err.Error(), "not supported for learner") { logrus.Debug("skipping bootstrap data save on learner") From b4401296ec3d7cc04e91cac6c7859deca171755d Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 21 Jul 2021 22:18:56 +0200 Subject: [PATCH 6/8] replace error with warn in delete Signed-off-by: galal-hussein --- pkg/cluster/storage.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 087a00bc8e..0f907f0885 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -226,5 +226,8 @@ func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue c } logrus.Infof("created bootstrap key %s", newTokenKey) // deleting the old key - return storageClient.Delete(ctx, oldTokenKey, keyValue.Modified) + if err := storageClient.Delete(ctx, oldTokenKey, keyValue.Modified); err != nil { + logrus.Warnf("failed to delete old bootstrap key %s", oldTokenKey) + } + return nil } From 7ebcc4b134ac0e992354f5db55f03b5ec4083b6f Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 21 Jul 2021 22:39:44 +0200 Subject: [PATCH 7/8] more fixes Signed-off-by: galal-hussein --- pkg/cluster/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 0f907f0885..16fcb39fcd 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -216,7 +216,7 @@ func doMigrateToken(ctx context.Context, storageClient client.Client, keyValue c // saving the new encrypted data with the right token key if err := storageClient.Create(ctx, newTokenKey, encryptedData); err != nil { if err.Error() == "key exists" { - logrus.Warnln("bootstrap key exists") + logrus.Warn("bootstrap key exists") } else if strings.Contains(err.Error(), "not supported for learner") { logrus.Debug("skipping bootstrap data save on learner") return nil From 20a48734c293a1b233d5bb5ca959a31e912496dc Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 21 Jul 2021 22:42:05 +0200 Subject: [PATCH 8/8] more fixes Signed-off-by: galal-hussein --- pkg/cluster/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 16fcb39fcd..a84a7f0886 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -142,7 +142,7 @@ func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient return nil, err } for _, bootstrapKV := range bootstrapList { - // checking for empty string bootstrap key + // ensure bootstrap is stored in the current token's key if string(bootstrapKV.Key) == tokenKey { return &bootstrapKV, nil }