Mirror/k3s
1
0
Fork 0
mirror of https://github.com/k3s-io/k3s.git synced 2024-06-07 19:41:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

[master] Add etcd extra args support for K3s (#4463)

Browse Source 
* Add etcd extra args support for K3s

Signed-off-by: Chris Kim <oats87g@gmail.com>

* Add etcd custom argument integration test

Signed-off-by: Chris Kim <oats87g@gmail.com>

* go generate

Signed-off-by: Chris Kim <oats87g@gmail.com>
This commit is contained in:
Chris Kim 2021-11-11 21:03:15 -08:00 committed by GitHub
parent 41ff19de71
commit f18b3252c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 127 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go
View File

@ -1,4 +1,3 @@
//go:build !ignore_autogenerated
// +build !ignore_autogenerated
/*

7
pkg/cli/cmds/server.go
View File

@ -48,6 +48,7 @@ type Server struct {
TLSSan cli.StringSlice
BindAddress string
ExtraAPIArgs cli.StringSlice
ExtraEtcdArgs cli.StringSlice
ExtraSchedulerArgs cli.StringSlice
ExtraControllerArgs cli.StringSlice
ExtraCloudControllerArgs cli.StringSlice
@ -128,6 +129,11 @@ var (
Usage: "(flags) Customized flag for kube-apiserver process",
Value: &ServerConfig.ExtraAPIArgs,
}
ExtraEtcdArgs = cli.StringSliceFlag{
Name: "etcd-arg",
Usage: "(flags) Customized flag for etcd process",
Value: &ServerConfig.ExtraEtcdArgs,
}
ExtraSchedulerArgs = cli.StringSliceFlag{
Name: "kube-scheduler-arg",
Usage: "(flags) Customized flag for kube-scheduler process",
@ -214,6 +220,7 @@ var ServerFlags = []cli.Flag{
EnvVar: version.ProgramUpper + "_KUBECONFIG_MODE",
},
ExtraAPIArgs,
ExtraEtcdArgs,
ExtraControllerArgs,
ExtraSchedulerArgs,
cli.StringSliceFlag{

1
pkg/cli/server/server.go
View File

@ -117,6 +117,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.APIServerBindAddress = cfg.APIServerBindAddress
serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
serverConfig.ControlConfig.ExtraEtcdArgs = cfg.ExtraEtcdArgs
serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
serverConfig.ControlConfig.ClusterDomain = cfg.ClusterDomain
serverConfig.ControlConfig.Datastore.Endpoint = cfg.DatastoreEndpoint

2
pkg/cluster/bootstrap.go
View File

@ -74,7 +74,7 @@ func (c *Cluster) Bootstrap(ctx context.Context, snapshot bool) error {
ElectionTimeout: 5000,
LogOutputs: []string{"stderr"},
}
configFile, err := args.ToConfigFile()
configFile, err := args.ToConfigFile(c.config.ExtraEtcdArgs)
if err != nil {
return err
}

1
pkg/daemons/config/types.go
View File

@ -135,6 +135,7 @@ type Control struct {
ExtraAPIArgs []string
ExtraControllerArgs []string
ExtraCloudControllerArgs []string
ExtraEtcdArgs []string
ExtraSchedulerAPIArgs []string
NoLeaderElect bool
JoinURL string

4
pkg/daemons/executor/etcd.go
View File

@ -19,8 +19,8 @@ func (e Embedded) CurrentETCDOptions() (InitialOptions, error) {
return InitialOptions{}, nil
}
func (e Embedded) ETCD(ctx context.Context, args ETCDConfig) error {
configFile, err := args.ToConfigFile()
func (e Embedded) ETCD(ctx context.Context, args ETCDConfig, extraArgs []string) error {
configFile, err := args.ToConfigFile(extraArgs)
if err != nil {
return err
}

55
pkg/daemons/executor/executor.go
View File

@ -6,12 +6,15 @@ import (
"net/http"
"os"
"path/filepath"
"sigs.k8s.io/yaml"
"strconv"
"strings"
"time"
"github.com/rancher/k3s/pkg/cli/cmds"
daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
yaml2 "gopkg.in/yaml.v2"
"k8s.io/apiserver/pkg/authentication/authenticator"
"sigs.k8s.io/yaml"
)
var (
@ -27,7 +30,7 @@ type Executor interface {
Scheduler(ctx context.Context, apiReady <-chan struct{}, args []string) error
ControllerManager(ctx context.Context, apiReady <-chan struct{}, args []string) error
CurrentETCDOptions() (InitialOptions, error)
ETCD(ctx context.Context, args ETCDConfig) error
ETCD(ctx context.Context, args ETCDConfig, extraArgs []string) error
CloudControllerManager(ctx context.Context, ccmRBACReady <-chan struct{}, args []string) error
}
@ -69,13 +72,53 @@ type InitialOptions struct {
State string `json:"initial-cluster-state,omitempty"`
}
func (e ETCDConfig) ToConfigFile() (string, error) {
func (e ETCDConfig) ToConfigFile(extraArgs []string) (string, error) {
confFile := filepath.Join(e.DataDir, "config")
bytes, err := yaml.Marshal(&e)
if err != nil {
return "", err
}
if len(extraArgs) > 0 {
var s map[string]interface{}
if err := yaml2.Unmarshal(bytes, &s); err != nil {
return "", err
}
for _, v := range extraArgs {
extraArg := strings.SplitN(v, "=", 2)
// Depending on the argV, we have different types to handle.
// Source: https://github.com/etcd-io/etcd/blob/44b8ae145b505811775f5af915dd19198d556d55/server/config/config.go#L36-L190 and https://etcd.io/docs/v3.5/op-guide/configuration/#configuration-file
if len(extraArg) == 2 {
key := strings.TrimLeft(extraArg[0], "-")
lowerKey := strings.ToLower(key)
var stringArr []string
if i, err := strconv.Atoi(extraArg[1]); err == nil {
s[key] = i
} else if time, err := time.ParseDuration(extraArg[1]); err == nil && (strings.Contains(lowerKey, "time") || strings.Contains(lowerKey, "duration") || strings.Contains(lowerKey, "interval") || strings.Contains(lowerKey, "retention")) {
// auto-compaction-retention is either a time.Duration or int, depending on version. If it is an int, it will be caught above.
s[key] = time
} else if err := yaml.Unmarshal([]byte(extraArg[1]), &stringArr); err == nil {
s[key] = stringArr
} else {
switch strings.ToLower(extraArg[1]) {
case "true":
s[key] = true
case "false":
s[key] = false
default:
s[key] = extraArg[1]
}
}
}
}
bytes, err = yaml2.Marshal(&s)
if err != nil {
return "", err
}
}
if err := os.MkdirAll(e.DataDir, 0700); err != nil {
return "", err
}
@ -118,8 +161,8 @@ func CurrentETCDOptions() (InitialOptions, error) {
return executor.CurrentETCDOptions()
}
func ETCD(ctx context.Context, args ETCDConfig) error {
return executor.ETCD(ctx, args)
func ETCD(ctx context.Context, args ETCDConfig, extraArgs []string) error {
return executor.ETCD(ctx, args, extraArgs)
}
func CloudControllerManager(ctx context.Context, ccmRBACReady <-chan struct{}, args []string) error {

1
pkg/deploy/zz_generated_bindata.go
View File

@ -12,7 +12,6 @@
// manifests/metrics-server/resource-reader.yaml
// manifests/rolebindings.yaml
// manifests/traefik.yaml
//go:build !no_stage
// +build !no_stage
package deploy

2
pkg/etcd/etcd.go
View File

@ -669,7 +669,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
HeartbeatInterval: 500,
Logger: "zap",
LogOutputs: []string{"stderr"},
})
}, e.config.ExtraEtcdArgs)
}
// RemovePeer removes a peer from the cluster. The peer name and IP address must both match.

5
pkg/static/zz_generated_bindata.go
View File

File diff suppressed because one or more lines are too long

63
tests/integration/custometcdargs/custometcdargs_int_test.go Normal file
View File

@ -0,0 +1,63 @@
package integration
import (
"os/exec"
"strings"
"testing"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/reporters"
. "github.com/onsi/gomega"
testutil "github.com/rancher/k3s/tests/util"
)
var customEtcdArgsServer *testutil.K3sServer
var customEtcdArgsServerArgs = []string{
"--cluster-init",
"--etcd-arg quota-backend-bytes=858993459",
}
var _ = BeforeSuite(func() {
if !testutil.IsExistingServer() {
var err error
customEtcdArgsServer, err = testutil.K3sStartServer(customEtcdArgsServerArgs...)
Expect(err).ToNot(HaveOccurred())
}
})
var _ = Describe("custom etcd args", func() {
BeforeEach(func() {
if testutil.IsExistingServer() && !testutil.ServerArgsPresent(customEtcdArgsServerArgs) {
Skip("Test needs k3s server with: " + strings.Join(customEtcdArgsServerArgs, " "))
}
})
When("a custom quota backend bytes is specified", func() {
It("renders a config file with the correct entry", func() {
Eventually(func() (string, error) {
var cmd *exec.Cmd
grepCmd := "grep"
grepCmdArgs := []string{"quota-backend-bytes", "/var/lib/rancher/k3s/server/db/etcd/config"}
if testutil.IsRoot() {
cmd = exec.Command(grepCmd, grepCmdArgs...)
} else {
fullGrepCmd := append([]string{grepCmd}, grepCmdArgs...)
cmd = exec.Command("sudo", fullGrepCmd...)
}
byteOut, err := cmd.CombinedOutput()
return string(byteOut), err
}, "45s", "5s").Should(MatchRegexp(".*quota-backend-bytes: 858993459.*"))
})
})
})
var _ = AfterSuite(func() {
if !testutil.IsExistingServer() {
Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed())
}
})
func Test_IntegrationCustomEtcdArgs(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t, "Custom etcd Arguments", []Reporter{
reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"),
})
}