From 65cd60683248ec10615a283c72c328e9916a12f1 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Mon, 11 Mar 2024 18:41:03 +0000 Subject: [PATCH] Respect cloud-provider fields set by kubelet Don't clobber the providerID field and instance-type/region/zone labels if provided by the kubelet. This allows the user to set these to the correct values when using the embedded CCM in a real cloud environment. Signed-off-by: Brad Davidson --- pkg/cloudprovider/instances.go | 41 ++++++--- pkg/cloudprovider/instances_test.go | 132 ++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 pkg/cloudprovider/instances_test.go diff --git a/pkg/cloudprovider/instances.go b/pkg/cloudprovider/instances.go index 2e399ae5c6..8afc29a14f 100644 --- a/pkg/cloudprovider/instances.go +++ b/pkg/cloudprovider/instances.go @@ -38,15 +38,34 @@ func (k *k3s) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudpr return nil, errors.New("address annotations not yet set") } - addresses := []corev1.NodeAddress{} + metadata := &cloudprovider.InstanceMetadata{ + ProviderID: fmt.Sprintf("%s://%s", version.Program, node.Name), + InstanceType: version.Program, + } + + if node.Spec.ProviderID != "" { + metadata.ProviderID = node.Spec.ProviderID + } + + if instanceType := node.Labels[corev1.LabelInstanceTypeStable]; instanceType != "" { + metadata.InstanceType = instanceType + } + + if region := node.Labels[corev1.LabelTopologyRegion]; region != "" { + metadata.Region = region + } + + if zone := node.Labels[corev1.LabelTopologyZone]; zone != "" { + metadata.Zone = zone + } // check internal address if address := node.Annotations[InternalIPKey]; address != "" { for _, v := range strings.Split(address, ",") { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: v}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: v}) } } else if address = node.Labels[InternalIPKey]; address != "" { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address}) } else { logrus.Infof("Couldn't find node internal ip annotation or label on node %s", node.Name) } @@ -54,26 +73,20 @@ func (k *k3s) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudpr // check external address if address := node.Annotations[ExternalIPKey]; address != "" { for _, v := range strings.Split(address, ",") { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: v}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: v}) } } else if address = node.Labels[ExternalIPKey]; address != "" { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: address}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: address}) } // check hostname if address := node.Annotations[HostnameKey]; address != "" { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address}) } else if address = node.Labels[HostnameKey]; address != "" { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address}) } else { logrus.Infof("Couldn't find node hostname annotation or label on node %s", node.Name) } - return &cloudprovider.InstanceMetadata{ - ProviderID: fmt.Sprintf("%s://%s", version.Program, node.Name), - InstanceType: version.Program, - NodeAddresses: addresses, - Zone: "", - Region: "", - }, nil + return metadata, nil } diff --git a/pkg/cloudprovider/instances_test.go b/pkg/cloudprovider/instances_test.go new file mode 100644 index 0000000000..379836e4a3 --- /dev/null +++ b/pkg/cloudprovider/instances_test.go @@ -0,0 +1,132 @@ +package cloudprovider + +import ( + "context" + "reflect" + "testing" + + "github.com/k3s-io/k3s/pkg/version" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cloudprovider "k8s.io/cloud-provider" +) + +func Test_UnitK3sInstanceMetadata(t *testing.T) { + nodeName := "test-node" + nodeInternalIP := "10.0.0.1" + nodeExternalIP := "1.2.3.4" + + tests := []struct { + name string + node *corev1.Node + want *cloudprovider.InstanceMetadata + wantErr bool + }{ + { + name: "No Annotations", + node: &corev1.Node{}, + wantErr: true, + }, + { + name: "Internal IP", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + InternalIPKey: nodeInternalIP, + }, + }, + }, + want: &cloudprovider.InstanceMetadata{ + InstanceType: version.Program, + ProviderID: version.Program + "://" + nodeName, + NodeAddresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: nodeInternalIP}, + }, + }, + }, + { + name: "Internal IP, External IP", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + InternalIPKey: nodeInternalIP, + ExternalIPKey: nodeExternalIP, + }, + }, + }, + want: &cloudprovider.InstanceMetadata{ + InstanceType: version.Program, + ProviderID: version.Program + "://" + nodeName, + NodeAddresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: nodeInternalIP}, + {Type: corev1.NodeExternalIP, Address: nodeExternalIP}, + }, + }, + }, + { + name: "Internal IP, External IP, Hostname", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + InternalIPKey: nodeInternalIP, + ExternalIPKey: nodeExternalIP, + HostnameKey: nodeName + ".example.com", + }, + }, + }, + want: &cloudprovider.InstanceMetadata{ + InstanceType: version.Program, + ProviderID: version.Program + "://" + nodeName, + NodeAddresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: nodeInternalIP}, + {Type: corev1.NodeExternalIP, Address: nodeExternalIP}, + {Type: corev1.NodeHostName, Address: nodeName + ".example.com"}, + }, + }, + }, + { + name: "Custom Metadata", + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + InternalIPKey: nodeInternalIP, + }, + Labels: map[string]string{ + corev1.LabelInstanceTypeStable: "test.t1", + corev1.LabelTopologyRegion: "region", + corev1.LabelTopologyZone: "zone", + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: "test://i-abc", + }, + }, + want: &cloudprovider.InstanceMetadata{ + InstanceType: "test.t1", + ProviderID: "test://i-abc", + NodeAddresses: []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: nodeInternalIP}, + }, + Region: "region", + Zone: "zone", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k := &k3s{} + got, err := k.InstanceMetadata(context.Background(), tt.node) + if (err != nil) != tt.wantErr { + t.Errorf("k3s.InstanceMetadata() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("k3s.InstanceMetadata() = %+v\nWant = %+v", got, tt.want) + } + }) + } +}