Mirror/k3s
1
0
Fork 0
mirror of https://github.com/k3s-io/k3s.git synced 2024-06-07 19:41:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add unit tests for pkg/etcd (#3549)

Browse Source 
* Created new etcd unit tests and testing support file

Signed-off-by: dereknola <derek.nola@suse.com>
This commit is contained in:
Derek Nola 2021-07-01 16:08:35 -07:00 committed by GitHub
parent cbfe673c43
commit c833183517
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 426 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/unitcoverage.yaml vendored
View File

@ -11,6 +11,8 @@ jobs:
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: '1.16.5'
- name: Checkout
uses: actions/checkout@v2
with:
@ -19,6 +21,10 @@ jobs:
run: |
go test ./pkg/... -coverprofile coverage.out
go tool cover -func coverage.out
- name: On Failure, Launch Debug Session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
timeout-minutes: 2
- name: Upload Results To Codecov
uses: codecov/codecov-action@v1
with:

61
pkg/daemons/control/deps/deps.go
View File

@ -90,6 +90,67 @@ func KubeConfig(dest, url, caCert, clientCert, clientKey string) error {
return kubeconfigTemplate.Execute(output, &data)
}
// FillRuntimeCerts is responsible for filling out all the
// .crt and .key filenames for a ControlRuntime.
func FillRuntimeCerts(config *config.Control, runtime *config.ControlRuntime) {
runtime.ClientCA = filepath.Join(config.DataDir, "tls", "client-ca.crt")
runtime.ClientCAKey = filepath.Join(config.DataDir, "tls", "client-ca.key")
runtime.ServerCA = filepath.Join(config.DataDir, "tls", "server-ca.crt")
runtime.ServerCAKey = filepath.Join(config.DataDir, "tls", "server-ca.key")
runtime.RequestHeaderCA = filepath.Join(config.DataDir, "tls", "request-header-ca.crt")
runtime.RequestHeaderCAKey = filepath.Join(config.DataDir, "tls", "request-header-ca.key")
runtime.IPSECKey = filepath.Join(config.DataDir, "cred", "ipsec.psk")
runtime.ServiceKey = filepath.Join(config.DataDir, "tls", "service.key")
runtime.PasswdFile = filepath.Join(config.DataDir, "cred", "passwd")
runtime.NodePasswdFile = filepath.Join(config.DataDir, "cred", "node-passwd")
runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig")
runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig")
runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig")
runtime.KubeConfigAPIServer = filepath.Join(config.DataDir, "cred", "api-server.kubeconfig")
runtime.KubeConfigCloudController = filepath.Join(config.DataDir, "cred", "cloud-controller.kubeconfig")
runtime.ClientAdminCert = filepath.Join(config.DataDir, "tls", "client-admin.crt")
runtime.ClientAdminKey = filepath.Join(config.DataDir, "tls", "client-admin.key")
runtime.ClientControllerCert = filepath.Join(config.DataDir, "tls", "client-controller.crt")
runtime.ClientControllerKey = filepath.Join(config.DataDir, "tls", "client-controller.key")
runtime.ClientCloudControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.crt")
runtime.ClientCloudControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.key")
runtime.ClientSchedulerCert = filepath.Join(config.DataDir, "tls", "client-scheduler.crt")
runtime.ClientSchedulerKey = filepath.Join(config.DataDir, "tls", "client-scheduler.key")
runtime.ClientKubeAPICert = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.crt")
runtime.ClientKubeAPIKey = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.key")
runtime.ClientKubeProxyCert = filepath.Join(config.DataDir, "tls", "client-kube-proxy.crt")
runtime.ClientKubeProxyKey = filepath.Join(config.DataDir, "tls", "client-kube-proxy.key")
runtime.ClientK3sControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.crt")
runtime.ClientK3sControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.key")
runtime.ServingKubeAPICert = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
runtime.ServingKubeAPIKey = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
runtime.ClientKubeletKey = filepath.Join(config.DataDir, "tls", "client-kubelet.key")
runtime.ServingKubeletKey = filepath.Join(config.DataDir, "tls", "serving-kubelet.key")
runtime.ClientAuthProxyCert = filepath.Join(config.DataDir, "tls", "client-auth-proxy.crt")
runtime.ClientAuthProxyKey = filepath.Join(config.DataDir, "tls", "client-auth-proxy.key")
runtime.ETCDServerCA = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.crt")
runtime.ETCDServerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.key")
runtime.ETCDPeerCA = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.crt")
runtime.ETCDPeerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.key")
runtime.ServerETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "server-client.crt")
runtime.ServerETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "server-client.key")
runtime.PeerServerClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.crt")
runtime.PeerServerClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.key")
runtime.ClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "client.crt")
runtime.ClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "client.key")
if config.EncryptSecrets {
runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json")
}
}
// GenServerDeps is responsible for generating the cluster dependencies
// needed to successfully bootstrap a cluster.
func GenServerDeps(config *config.Control, runtime *config.ControlRuntime) error {

57
pkg/daemons/control/server.go
View File

@ -248,62 +248,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
os.MkdirAll(filepath.Join(config.DataDir, "tls"), 0700)
os.MkdirAll(filepath.Join(config.DataDir, "cred"), 0700)
runtime.ClientCA = filepath.Join(config.DataDir, "tls", "client-ca.crt")
runtime.ClientCAKey = filepath.Join(config.DataDir, "tls", "client-ca.key")
runtime.ServerCA = filepath.Join(config.DataDir, "tls", "server-ca.crt")
runtime.ServerCAKey = filepath.Join(config.DataDir, "tls", "server-ca.key")
runtime.RequestHeaderCA = filepath.Join(config.DataDir, "tls", "request-header-ca.crt")
runtime.RequestHeaderCAKey = filepath.Join(config.DataDir, "tls", "request-header-ca.key")
runtime.IPSECKey = filepath.Join(config.DataDir, "cred", "ipsec.psk")
runtime.ServiceKey = filepath.Join(config.DataDir, "tls", "service.key")
runtime.PasswdFile = filepath.Join(config.DataDir, "cred", "passwd")
runtime.NodePasswdFile = filepath.Join(config.DataDir, "cred", "node-passwd")
runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig")
runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig")
runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig")
runtime.KubeConfigAPIServer = filepath.Join(config.DataDir, "cred", "api-server.kubeconfig")
runtime.KubeConfigCloudController = filepath.Join(config.DataDir, "cred", "cloud-controller.kubeconfig")
runtime.ClientAdminCert = filepath.Join(config.DataDir, "tls", "client-admin.crt")
runtime.ClientAdminKey = filepath.Join(config.DataDir, "tls", "client-admin.key")
runtime.ClientControllerCert = filepath.Join(config.DataDir, "tls", "client-controller.crt")
runtime.ClientControllerKey = filepath.Join(config.DataDir, "tls", "client-controller.key")
runtime.ClientCloudControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.crt")
runtime.ClientCloudControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.key")
runtime.ClientSchedulerCert = filepath.Join(config.DataDir, "tls", "client-scheduler.crt")
runtime.ClientSchedulerKey = filepath.Join(config.DataDir, "tls", "client-scheduler.key")
runtime.ClientKubeAPICert = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.crt")
runtime.ClientKubeAPIKey = filepath.Join(config.DataDir, "tls", "client-kube-apiserver.key")
runtime.ClientKubeProxyCert = filepath.Join(config.DataDir, "tls", "client-kube-proxy.crt")
runtime.ClientKubeProxyKey = filepath.Join(config.DataDir, "tls", "client-kube-proxy.key")
runtime.ClientK3sControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.crt")
runtime.ClientK3sControllerKey = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-controller.key")
runtime.ServingKubeAPICert = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
runtime.ServingKubeAPIKey = filepath.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
runtime.ClientKubeletKey = filepath.Join(config.DataDir, "tls", "client-kubelet.key")
runtime.ServingKubeletKey = filepath.Join(config.DataDir, "tls", "serving-kubelet.key")
runtime.ClientAuthProxyCert = filepath.Join(config.DataDir, "tls", "client-auth-proxy.crt")
runtime.ClientAuthProxyKey = filepath.Join(config.DataDir, "tls", "client-auth-proxy.key")
runtime.ETCDServerCA = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.crt")
runtime.ETCDServerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "server-ca.key")
runtime.ETCDPeerCA = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.crt")
runtime.ETCDPeerCAKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-ca.key")
runtime.ServerETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "server-client.crt")
runtime.ServerETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "server-client.key")
runtime.PeerServerClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.crt")
runtime.PeerServerClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "peer-server-client.key")
runtime.ClientETCDCert = filepath.Join(config.DataDir, "tls", "etcd", "client.crt")
runtime.ClientETCDKey = filepath.Join(config.DataDir, "tls", "etcd", "client.key")
if config.EncryptSecrets {
runtime.EncryptionConfig = filepath.Join(config.DataDir, "cred", "encryption-config.json")
}
deps.FillRuntimeCerts(config, runtime)
cluster := cluster.New(config)

299
pkg/etcd/etcd_test.go Normal file
View File

@ -0,0 +1,299 @@
package etcd
import (
"context"
"net"
"net/http"
"os"
"path/filepath"
"testing"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/util/tests"
"github.com/robfig/cron/v3"
etcd "go.etcd.io/etcd/clientv3"
)
func generateTestConfig() *config.Control {
_, clusterIPNet, _ := net.ParseCIDR("10.42.0.0/16")
_, serviceIPNet, _ := net.ParseCIDR("10.43.0.0/16")
return &config.Control{
HTTPSPort: 6443,
SupervisorPort: 6443,
AdvertisePort: 6443,
ClusterDomain: "cluster.local",
ClusterDNS: net.ParseIP("10.43.0.10"),
ClusterIPRange: clusterIPNet,
DataDir: "/tmp/k3s/", // Different than the default value
FlannelBackend: "vxlan",
EtcdSnapshotName: "etcd-snapshot",
EtcdSnapshotCron: "0 */12 * * *",
EtcdSnapshotRetention: 5,
EtcdS3Endpoint: "s3.amazonaws.com",
EtcdS3Region: "us-east-1",
SANs: []string{"127.0.0.1"},
ServiceIPRange: serviceIPNet,
}
}
func generateTestHandler() http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {})
}
func TestETCD_IsInitialized(t *testing.T) {
type args struct {
ctx context.Context
config *config.Control
}
tests := []struct {
name string
args args
setup func(*config.Control) error
teardown func(*config.Control) error
want bool
wantErr bool
}{
{
name: "Directory exists",
args: args{
ctx: context.TODO(),
config: generateTestConfig(),
},
setup: func(cnf *config.Control) error {
if err := tests.GenerateDataDir(cnf); err != nil {
return err
}
return os.MkdirAll(walDir(cnf), 0700)
},
teardown: func(cnf *config.Control) error {
tests.CleanupDataDir(cnf)
return os.Remove(walDir(cnf))
},
wantErr: false,
want: true,
},
{
name: "Directory does not exist",
args: args{
ctx: context.TODO(),
config: generateTestConfig(),
},
setup: func(cnf *config.Control) error {
if err := tests.GenerateDataDir(cnf); err != nil {
return err
}
// We don't care if removal fails to find the dir
os.Remove(walDir(cnf))
return nil
},
teardown: func(cnf *config.Control) error {
tests.CleanupDataDir(cnf)
return nil
},
wantErr: false,
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
e := NewETCD()
defer tt.teardown(tt.args.config)
if err := tt.setup(tt.args.config); err != nil {
t.Errorf("Prep for ETCD.IsInitialized() failed = %v", err)
return
}
got, err := e.IsInitialized(tt.args.ctx, tt.args.config)
if (err != nil) != tt.wantErr {
t.Errorf("ETCD.IsInitialized() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("ETCD.IsInitialized() = %+v\nWant = %+v", got, tt.want)
return
}
})
}
}
func TestETCD_Register(t *testing.T) {
type args struct {
ctx context.Context
config *config.Control
handler http.Handler
}
tests := []struct {
name string
args args
setup func(cnf *config.Control) error
teardown func(cnf *config.Control) error
wantErr bool
}{
{
name: "Call Register with standard config",
args: args{
ctx: context.TODO(),
config: generateTestConfig(),
handler: generateTestHandler(),
},
setup: func(cnf *config.Control) error {
return tests.GenerateRuntime(cnf)
},
teardown: func(cnf *config.Control) error {
tests.CleanupDataDir(cnf)
return nil
},
},
{
name: "Call Register with a tombstone file created",
args: args{
ctx: context.TODO(),
config: generateTestConfig(),
handler: generateTestHandler(),
},
setup: func(cnf *config.Control) error {
if err := tests.GenerateRuntime(cnf); err != nil {
return err
}
if err := os.MkdirAll(etcdDBDir(cnf), 0700); err != nil {
return err
}
tombstoneFile := filepath.Join(etcdDBDir(cnf), "tombstone")
if _, err := os.Create(tombstoneFile); err != nil {
return err
}
return nil
},
teardown: func(cnf *config.Control) error {
tombstoneFile := filepath.Join(etcdDBDir(cnf), "tombstone")
os.Remove(tombstoneFile)
tests.CleanupDataDir(cnf)
return nil
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
e := NewETCD()
defer tt.teardown(tt.args.config)
if err := tt.setup(tt.args.config); err != nil {
t.Errorf("Setup for ETCD.Register() failed = %v", err)
return
}
_, err := e.Register(tt.args.ctx, tt.args.config, tt.args.handler)
if (err != nil) != tt.wantErr {
t.Errorf("ETCD.Register() error = %v, wantErr %v", err, tt.wantErr)
return
}
})
}
}
func TestETCD_Start(t *testing.T) {
type fields struct {
client *etcd.Client
config *config.Control
name string
runtime *config.ControlRuntime
address string
cron *cron.Cron
s3 *S3
}
type args struct {
ctx context.Context
clientAccessInfo *clientaccess.Info
}
tests := []struct {
name string
fields fields
args args
setup func(cnf *config.Control) error
teardown func(cnf *config.Control) error
wantErr bool
}{
{
name: "Start etcd without clientAccessInfo and without snapshots",
fields: fields{
config: generateTestConfig(),
address: "192.168.1.123", // Local IP address
},
args: args{
ctx: context.TODO(),
clientAccessInfo: nil,
},
setup: func(cnf *config.Control) error {
cnf.EtcdDisableSnapshots = true
return tests.GenerateRuntime(cnf)
},
teardown: func(cnf *config.Control) error {
tests.CleanupDataDir(cnf)
return nil
},
},
{
name: "Start etcd without clientAccessInfo on",
fields: fields{
config: generateTestConfig(),
address: "192.168.1.123", // Local IP address
cron: cron.New(),
},
args: args{
ctx: context.TODO(),
clientAccessInfo: nil,
},
setup: func(cnf *config.Control) error {
return tests.GenerateRuntime(cnf)
},
teardown: func(cnf *config.Control) error {
tests.CleanupDataDir(cnf)
return nil
},
},
{
name: "Start etcd with an existing cluster",
fields: fields{
config: generateTestConfig(),
address: "192.168.1.123", // Local IP address
cron: cron.New(),
},
args: args{
ctx: context.TODO(),
clientAccessInfo: nil,
},
setup: func(cnf *config.Control) error {
if err := tests.GenerateRuntime(cnf); err != nil {
return err
}
return os.MkdirAll(walDir(cnf), 0700)
},
teardown: func(cnf *config.Control) error {
tests.CleanupDataDir(cnf)
os.Remove(walDir(cnf))
return nil
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
e := &ETCD{
client: tt.fields.client,
config: tt.fields.config,
name: tt.fields.name,
runtime: tt.fields.runtime,
address: tt.fields.address,
cron: tt.fields.cron,
s3: tt.fields.s3,
}
defer tt.teardown(e.config)
if err := tt.setup(e.config); err != nil {
t.Errorf("Setup for ETCD.Start() failed = %v", err)
return
}
if err := e.Start(tt.args.ctx, tt.args.clientAccessInfo); (err != nil) != tt.wantErr {
t.Errorf("ETCD.Start() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

59
pkg/util/tests/runtime.go Normal file
View File

@ -0,0 +1,59 @@
package tests
import (
"os"
"path/filepath"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/daemons/control/deps"
)
// GenerateDataDir creates a temporary directory at "/tmp/k3s/<RANDOM_STRING>/".
// The latest directory created with this function is soft linked to "/tmp/k3s/latest/".
// This allows tests to replicate the "/var/lib/rancher/k3s" directory structure.
func GenerateDataDir(cnf *config.Control) error {
if err := os.MkdirAll(cnf.DataDir, 0700); err != nil {
return err
}
testDir, err := os.MkdirTemp(cnf.DataDir, "*")
if err != nil {
return err
}
// Remove old symlink and add new one
os.Remove(filepath.Join(cnf.DataDir, "latest"))
if err = os.Symlink(testDir, filepath.Join(cnf.DataDir, "latest")); err != nil {
return err
}
cnf.DataDir = testDir
cnf.DataDir, err = filepath.Abs(cnf.DataDir)
if err != nil {
return err
}
return nil
}
// CleanupDataDir removes the associated "/tmp/k3s/<RANDOM_STRING>"
// directory.
func CleanupDataDir(cnf *config.Control) {
os.RemoveAll(cnf.DataDir)
}
// GenerateRuntime creates a temporary data dir and configures
// config.ControlRuntime with all the appropriate certificate keys.
func GenerateRuntime(cnf *config.Control) error {
runtime := &config.ControlRuntime{}
if err := GenerateDataDir(cnf); err != nil {
return err
}
os.MkdirAll(filepath.Join(cnf.DataDir, "tls"), 0700)
os.MkdirAll(filepath.Join(cnf.DataDir, "cred"), 0700)
deps.FillRuntimeCerts(cnf, runtime)
if err := deps.GenServerDeps(cnf, runtime); err != nil {
return err
}
cnf.Runtime = runtime
return nil
}