mirror of
https://github.com/k3s-io/k3s.git
synced 2024-06-07 19:41:36 +00:00
Add k3s cloud provider
This commit is contained in:
galal-hussein
parent
9961b1a625
commit
d2c1f66496
72
manifests/ccm.yaml
Normal file
72
manifests/ccm.yaml
Normal file
@ -0,0 +1,72 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: cloud-controller-manager
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- events
|
||||
verbs:
|
||||
- create
|
||||
- patch
|
||||
- update
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes
|
||||
verbs:
|
||||
- '*'
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- nodes/status
|
||||
verbs:
|
||||
- patch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- services
|
||||
verbs:
|
||||
- list
|
||||
- patch
|
||||
- update
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- serviceaccounts
|
||||
verbs:
|
||||
- create
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- persistentvolumes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- update
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- endpoints
|
||||
verbs:
|
||||
- create
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- update
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: cloud-controller-manager
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: cloud-controller-manager
|
||||
subjects:
|
||||
- kind: User
|
||||
name: cloud-controller-manager
|
||||
namespace: kube-system
|
||||
@ -337,6 +337,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
|
||||
nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
|
||||
nodeConfig.AgentConfig.NodeIP = nodeIP
|
||||
nodeConfig.AgentConfig.NodeName = nodeName
|
||||
nodeConfig.AgentConfig.NodeExternalIP = envInfo.NodeExternalIP
|
||||
nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
|
||||
nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey
|
||||
nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
|
||||
@ -398,6 +399,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
|
||||
nodeConfig.AgentConfig.NodeTaints = envInfo.Taints
|
||||
nodeConfig.AgentConfig.NodeLabels = envInfo.Labels
|
||||
nodeConfig.AgentConfig.PrivateRegistry = envInfo.PrivateRegistry
|
||||
nodeConfig.AgentConfig.DisableCCM = controlConfig.DisableCCM
|
||||
|
||||
return nodeConfig, nil
|
||||
}
|
||||
|
||||
@ -18,8 +18,19 @@ import (
|
||||
"github.com/rancher/k3s/pkg/cli/cmds"
|
||||
"github.com/rancher/k3s/pkg/clientaccess"
|
||||
"github.com/rancher/k3s/pkg/daemons/agent"
|
||||
daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
|
||||
"github.com/rancher/k3s/pkg/rootless"
|
||||
"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
|
||||
corev1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
|
||||
"github.com/rancher/wrangler/pkg/start"
|
||||
"github.com/sirupsen/logrus"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
)
|
||||
|
||||
const (
|
||||
InternalIPLabel = "k3s.io/internal-ip"
|
||||
ExternalIPLabel = "k3s.io/external-ip"
|
||||
HostnameLabel = "k3s.io/hostname"
|
||||
)
|
||||
|
||||
func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) error {
|
||||
@ -58,6 +69,12 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
|
||||
}
|
||||
}
|
||||
|
||||
if !nodeConfig.AgentConfig.DisableCCM {
|
||||
if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
@ -124,3 +141,74 @@ func validate() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent) error {
|
||||
for {
|
||||
nodeController, nodeCache, err := startNodeController(ctx, agentConfig)
|
||||
if err != nil {
|
||||
logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
|
||||
time.Sleep(1 * time.Second)
|
||||
continue
|
||||
}
|
||||
nodeCached, err := nodeCache.Get(agentConfig.NodeName)
|
||||
if err != nil {
|
||||
logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
|
||||
time.Sleep(1 * time.Second)
|
||||
continue
|
||||
}
|
||||
node := nodeCached.DeepCopy()
|
||||
updated := updateLabelMap(ctx, agentConfig, node.Labels)
|
||||
if updated {
|
||||
_, err = nodeController.Update(node)
|
||||
if err == nil {
|
||||
logrus.Infof("addresses labels has been set succesfully on node: %s", agentConfig.NodeName)
|
||||
break
|
||||
}
|
||||
logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
|
||||
time.Sleep(1 * time.Second)
|
||||
continue
|
||||
}
|
||||
logrus.Infof("addresses labels has already been set succesfully on node: %s", agentConfig.NodeName)
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func startNodeController(ctx context.Context, agentConfig *daemonconfig.Agent) (corev1.NodeController, corev1.NodeCache, error) {
|
||||
restConfig, err := clientcmd.BuildConfigFromFlags("", agentConfig.KubeConfigKubelet)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
coreFactory := core.NewFactoryFromConfigOrDie(restConfig)
|
||||
nodeController := coreFactory.Core().V1().Node()
|
||||
nodeCache := nodeController.Cache()
|
||||
if err := start.All(ctx, 1, coreFactory); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return nodeController, nodeCache, nil
|
||||
}
|
||||
|
||||
func updateLabelMap(ctx context.Context, agentConfig *daemonconfig.Agent, nodeLabels map[string]string) bool {
|
||||
if nodeLabels == nil {
|
||||
nodeLabels = make(map[string]string)
|
||||
}
|
||||
updated := false
|
||||
if internalIPLabel, ok := nodeLabels[InternalIPLabel]; !ok || internalIPLabel != agentConfig.NodeIP {
|
||||
nodeLabels[InternalIPLabel] = agentConfig.NodeIP
|
||||
updated = true
|
||||
}
|
||||
if hostnameLabel, ok := nodeLabels[HostnameLabel]; !ok || hostnameLabel != agentConfig.NodeName {
|
||||
nodeLabels[HostnameLabel] = agentConfig.NodeName
|
||||
updated = true
|
||||
}
|
||||
nodeExternalIP := agentConfig.NodeExternalIP
|
||||
if externalIPLabel := nodeLabels[ExternalIPLabel]; externalIPLabel != nodeExternalIP && nodeExternalIP != "" {
|
||||
nodeLabels[ExternalIPLabel] = nodeExternalIP
|
||||
updated = true
|
||||
} else if nodeExternalIP == "" && externalIPLabel != "" {
|
||||
delete(nodeLabels, ExternalIPLabel)
|
||||
updated = true
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
@ -15,6 +15,7 @@ type Agent struct {
|
||||
ResolvConf string
|
||||
DataDir string
|
||||
NodeIP string
|
||||
NodeExternalIP string
|
||||
NodeName string
|
||||
ClusterSecret string
|
||||
PauseImage string
|
||||
@ -45,6 +46,11 @@ var (
|
||||
Usage: "(agent) IP address to advertise for node",
|
||||
Destination: &AgentConfig.NodeIP,
|
||||
}
|
||||
NodeExternalIPFlag = cli.StringFlag{
|
||||
Name: "node-external-ip",
|
||||
Usage: "(agent) External IP address to advertise for node",
|
||||
Destination: &AgentConfig.NodeExternalIP,
|
||||
}
|
||||
NodeNameFlag = cli.StringFlag{
|
||||
Name: "node-name",
|
||||
Usage: "(agent) Node name",
|
||||
@ -175,6 +181,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
|
||||
NodeLabels,
|
||||
NodeTaints,
|
||||
PrivateRegistryFlag,
|
||||
NodeExternalIPFlag,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,33 +8,35 @@ import (
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
ClusterCIDR string
|
||||
ClusterSecret string
|
||||
ServiceCIDR string
|
||||
ClusterDNS string
|
||||
ClusterDomain string
|
||||
HTTPSPort int
|
||||
HTTPPort int
|
||||
DataDir string
|
||||
DisableAgent bool
|
||||
KubeConfigOutput string
|
||||
KubeConfigMode string
|
||||
TLSSan cli.StringSlice
|
||||
BindAddress string
|
||||
ExtraAPIArgs cli.StringSlice
|
||||
ExtraSchedulerArgs cli.StringSlice
|
||||
ExtraControllerArgs cli.StringSlice
|
||||
Rootless bool
|
||||
StoreBootstrap bool
|
||||
StorageEndpoint string
|
||||
StorageCAFile string
|
||||
StorageCertFile string
|
||||
StorageKeyFile string
|
||||
AdvertiseIP string
|
||||
AdvertisePort int
|
||||
DisableScheduler bool
|
||||
FlannelBackend string
|
||||
DefaultLocalStoragePath string
|
||||
ClusterCIDR string
|
||||
ClusterSecret string
|
||||
ServiceCIDR string
|
||||
ClusterDNS string
|
||||
ClusterDomain string
|
||||
HTTPSPort int
|
||||
HTTPPort int
|
||||
DataDir string
|
||||
DisableAgent bool
|
||||
KubeConfigOutput string
|
||||
KubeConfigMode string
|
||||
TLSSan cli.StringSlice
|
||||
BindAddress string
|
||||
ExtraAPIArgs cli.StringSlice
|
||||
ExtraSchedulerArgs cli.StringSlice
|
||||
ExtraControllerArgs cli.StringSlice
|
||||
ExtraCloudControllerArgs cli.StringSlice
|
||||
Rootless bool
|
||||
StoreBootstrap bool
|
||||
StorageEndpoint string
|
||||
StorageCAFile string
|
||||
StorageCertFile string
|
||||
StorageKeyFile string
|
||||
AdvertiseIP string
|
||||
AdvertisePort int
|
||||
DisableScheduler bool
|
||||
FlannelBackend string
|
||||
DefaultLocalStoragePath string
|
||||
DisableCCM bool
|
||||
}
|
||||
|
||||
var ServerConfig Server
|
||||
@ -143,6 +145,11 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
|
||||
Usage: "Customized flag for kube-controller-manager process",
|
||||
Value: &ServerConfig.ExtraControllerArgs,
|
||||
},
|
||||
cli.StringSliceFlag{
|
||||
Name: "kube-cloud-controller-arg",
|
||||
Usage: "Customized flag for kube-cloud-controller-manager process",
|
||||
Value: &ServerConfig.ExtraCloudControllerArgs,
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "rootless",
|
||||
Usage: "(experimental) Run rootless",
|
||||
@ -194,6 +201,11 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
|
||||
Usage: "Disable Kubernetes default scheduler",
|
||||
Destination: &ServerConfig.DisableScheduler,
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "disable-cloud-controller",
|
||||
Usage: "Disable k3s default cloud controller manager",
|
||||
Destination: &ServerConfig.DisableCCM,
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "flannel-backend",
|
||||
Usage: fmt.Sprintf("(experimental) One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard),
|
||||
@ -219,6 +231,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
|
||||
NodeLabels,
|
||||
NodeTaints,
|
||||
PrivateRegistryFlag,
|
||||
NodeExternalIPFlag,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@ -85,6 +85,8 @@ func run(app *cli.Context, cfg *cmds.Server) error {
|
||||
serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
|
||||
serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
|
||||
serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
|
||||
serverConfig.ControlConfig.ExtraCloudControllerArgs = cfg.ExtraCloudControllerArgs
|
||||
serverConfig.ControlConfig.DisableCCM = cfg.DisableCCM
|
||||
|
||||
if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
|
||||
cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)
|
||||
|
||||
57
pkg/cloudprovider/cloudprovider.go
Normal file
57
pkg/cloudprovider/cloudprovider.go
Normal file
@ -0,0 +1,57 @@
|
||||
package cloudprovider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
|
||||
coreclient "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
|
||||
"github.com/rancher/wrangler/pkg/start"
|
||||
cloudprovider "k8s.io/cloud-provider"
|
||||
)
|
||||
|
||||
type k3s struct {
|
||||
NodeCache coreclient.NodeCache
|
||||
}
|
||||
|
||||
func init() {
|
||||
cloudprovider.RegisterCloudProvider("k3s", func(config io.Reader) (cloudprovider.Interface, error) {
|
||||
return &k3s{}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (k *k3s) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) {
|
||||
coreFactory := core.NewFactoryFromConfigOrDie(clientBuilder.ConfigOrDie("cloud-controller-manager"))
|
||||
|
||||
go start.All(context.Background(), 1, coreFactory)
|
||||
|
||||
k.NodeCache = coreFactory.Core().V1().Node().Cache()
|
||||
}
|
||||
|
||||
func (k *k3s) Instances() (cloudprovider.Instances, bool) {
|
||||
return k, true
|
||||
}
|
||||
|
||||
func (k *k3s) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (k *k3s) Zones() (cloudprovider.Zones, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (k *k3s) Clusters() (cloudprovider.Clusters, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (k *k3s) Routes() (cloudprovider.Routes, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (k *k3s) ProviderName() string {
|
||||
return "k3s"
|
||||
}
|
||||
|
||||
func (k *k3s) HasClusterID() bool {
|
||||
return false
|
||||
}
|
||||
85
pkg/cloudprovider/instances.go
Normal file
85
pkg/cloudprovider/instances.go
Normal file
@ -0,0 +1,85 @@
|
||||
package cloudprovider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
cloudprovider "k8s.io/cloud-provider"
|
||||
)
|
||||
|
||||
const (
|
||||
InternalIPLabel = "k3s.io/internal-ip"
|
||||
ExternalIPLabel = "k3s.io/external-ip"
|
||||
HostnameLabel = "k3s.io/hostname"
|
||||
)
|
||||
|
||||
func (k *k3s) AddSSHKeyToAllInstances(ctx context.Context, user string, keyData []byte) error {
|
||||
return cloudprovider.NotImplemented
|
||||
}
|
||||
|
||||
func (k *k3s) CurrentNodeName(ctx context.Context, hostname string) (types.NodeName, error) {
|
||||
return types.NodeName(hostname), nil
|
||||
}
|
||||
|
||||
func (k *k3s) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (k *k3s) InstanceID(ctx context.Context, nodeName types.NodeName) (string, error) {
|
||||
_, err := k.NodeCache.Get(string(nodeName))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Failed to find node %s: %v", nodeName, err)
|
||||
}
|
||||
return string(nodeName), nil
|
||||
}
|
||||
|
||||
func (k *k3s) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) {
|
||||
return true, cloudprovider.NotImplemented
|
||||
}
|
||||
|
||||
func (k *k3s) InstanceType(ctx context.Context, name types.NodeName) (string, error) {
|
||||
_, err := k.InstanceID(ctx, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "k3s", nil
|
||||
}
|
||||
|
||||
func (k *k3s) InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) {
|
||||
return "", cloudprovider.NotImplemented
|
||||
}
|
||||
|
||||
func (k *k3s) NodeAddresses(ctx context.Context, name types.NodeName) ([]corev1.NodeAddress, error) {
|
||||
addresses := []corev1.NodeAddress{}
|
||||
node, err := k.NodeCache.Get(string(name))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to find node %s: %v", name, err)
|
||||
}
|
||||
// check internal address
|
||||
if node.Labels[InternalIPLabel] != "" {
|
||||
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: node.Labels[InternalIPLabel]})
|
||||
} else {
|
||||
logrus.Infof("couldn't find node internal ip label on node %s", name)
|
||||
}
|
||||
|
||||
// check external address
|
||||
if node.Labels[ExternalIPLabel] != "" {
|
||||
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: node.Labels[ExternalIPLabel]})
|
||||
}
|
||||
|
||||
// check hostname
|
||||
if node.Labels[HostnameLabel] != "" {
|
||||
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: node.Labels[HostnameLabel]})
|
||||
} else {
|
||||
logrus.Infof("couldn't find node hostname label on node %s", name)
|
||||
}
|
||||
|
||||
return addresses, nil
|
||||
}
|
||||
|
||||
func (k *k3s) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]corev1.NodeAddress, error) {
|
||||
return nil, cloudprovider.NotImplemented
|
||||
}
|
||||
@ -133,6 +133,10 @@ func kubelet(cfg *config.Agent) {
|
||||
if len(cfg.NodeTaints) > 0 {
|
||||
argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
|
||||
}
|
||||
if !cfg.DisableCCM {
|
||||
argsMap["cloud-provider"] = "external"
|
||||
}
|
||||
|
||||
args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
|
||||
command.SetArgs(args)
|
||||
|
||||
|
||||
@ -63,6 +63,7 @@ type Agent struct {
|
||||
KubeConfigKubelet string
|
||||
KubeConfigKubeProxy string
|
||||
NodeIP string
|
||||
NodeExternalIP string
|
||||
RuntimeSocket string
|
||||
ListenAddress string
|
||||
ClientCA string
|
||||
@ -77,33 +78,36 @@ type Agent struct {
|
||||
IPSECPSK string
|
||||
StrongSwanDir string
|
||||
PrivateRegistry string
|
||||
DisableCCM bool
|
||||
}
|
||||
|
||||
type Control struct {
|
||||
AdvertisePort int
|
||||
AdvertiseIP string
|
||||
ListenPort int
|
||||
HTTPSPort int
|
||||
ClusterSecret string
|
||||
ClusterIPRange *net.IPNet
|
||||
ServiceIPRange *net.IPNet
|
||||
ClusterDNS net.IP
|
||||
ClusterDomain string
|
||||
NoCoreDNS bool
|
||||
KubeConfigOutput string
|
||||
KubeConfigMode string
|
||||
DataDir string
|
||||
Skips []string
|
||||
BootstrapReadOnly bool
|
||||
Storage endpoint.Config
|
||||
NoScheduler bool
|
||||
ExtraAPIArgs []string
|
||||
ExtraControllerArgs []string
|
||||
ExtraSchedulerAPIArgs []string
|
||||
NoLeaderElect bool
|
||||
FlannelBackend string
|
||||
IPSECPSK string
|
||||
DefaultLocalStoragePath string
|
||||
AdvertisePort int
|
||||
AdvertiseIP string
|
||||
ListenPort int
|
||||
HTTPSPort int
|
||||
ClusterSecret string
|
||||
ClusterIPRange *net.IPNet
|
||||
ServiceIPRange *net.IPNet
|
||||
ClusterDNS net.IP
|
||||
ClusterDomain string
|
||||
NoCoreDNS bool
|
||||
KubeConfigOutput string
|
||||
KubeConfigMode string
|
||||
DataDir string
|
||||
Skips []string
|
||||
BootstrapReadOnly bool
|
||||
Storage endpoint.Config
|
||||
NoScheduler bool
|
||||
ExtraAPIArgs []string
|
||||
ExtraControllerArgs []string
|
||||
ExtraSchedulerAPIArgs []string
|
||||
ExtraCloudControllerArgs []string
|
||||
NoLeaderElect bool
|
||||
FlannelBackend string
|
||||
IPSECPSK string
|
||||
DefaultLocalStoragePath string
|
||||
DisableCCM bool
|
||||
|
||||
Runtime *ControlRuntime `json:"-"`
|
||||
}
|
||||
@ -130,10 +134,11 @@ type ControlRuntime struct {
|
||||
ClientKubeAPIKey string
|
||||
NodePasswdFile string
|
||||
|
||||
KubeConfigAdmin string
|
||||
KubeConfigController string
|
||||
KubeConfigScheduler string
|
||||
KubeConfigAPIServer string
|
||||
KubeConfigAdmin string
|
||||
KubeConfigController string
|
||||
KubeConfigScheduler string
|
||||
KubeConfigAPIServer string
|
||||
KubeConfigCloudController string
|
||||
|
||||
ServingKubeAPICert string
|
||||
ServingKubeAPIKey string
|
||||
@ -146,13 +151,15 @@ type ControlRuntime struct {
|
||||
ClientAuthProxyCert string
|
||||
ClientAuthProxyKey string
|
||||
|
||||
ClientAdminCert string
|
||||
ClientAdminKey string
|
||||
ClientControllerCert string
|
||||
ClientControllerKey string
|
||||
ClientSchedulerCert string
|
||||
ClientSchedulerKey string
|
||||
ClientKubeProxyCert string
|
||||
ClientAdminCert string
|
||||
ClientAdminKey string
|
||||
ClientControllerCert string
|
||||
ClientControllerKey string
|
||||
ClientSchedulerCert string
|
||||
ClientSchedulerKey string
|
||||
ClientKubeProxyCert string
|
||||
ClientCloudControllerCert string
|
||||
ClientCloudControllerKey string
|
||||
}
|
||||
|
||||
type ArgString []string
|
||||
|
||||
@ -22,11 +22,17 @@ import (
|
||||
"time"
|
||||
|
||||
certutil "github.com/rancher/dynamiclistener/cert"
|
||||
// registering k3s cloud provider
|
||||
_ "github.com/rancher/k3s/pkg/cloudprovider"
|
||||
"github.com/rancher/k3s/pkg/daemons/config"
|
||||
"github.com/rancher/kine/pkg/client"
|
||||
"github.com/rancher/kine/pkg/endpoint"
|
||||
"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
|
||||
"github.com/sirupsen/logrus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apiserver/pkg/authentication/authenticator"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
ccmapp "k8s.io/kubernetes/cmd/cloud-controller-manager/app"
|
||||
"k8s.io/kubernetes/cmd/kube-apiserver/app"
|
||||
cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
|
||||
sapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
|
||||
@ -94,6 +100,10 @@ func Server(ctx context.Context, cfg *config.Control) error {
|
||||
|
||||
controllerManager(cfg, runtime)
|
||||
|
||||
if !cfg.DisableCCM {
|
||||
cloudControllerManager(cfg, runtime)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -267,11 +277,14 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
|
||||
runtime.KubeConfigController = path.Join(config.DataDir, "cred", "controller.kubeconfig")
|
||||
runtime.KubeConfigScheduler = path.Join(config.DataDir, "cred", "scheduler.kubeconfig")
|
||||
runtime.KubeConfigAPIServer = path.Join(config.DataDir, "cred", "api-server.kubeconfig")
|
||||
runtime.KubeConfigCloudController = path.Join(config.DataDir, "cred", "cloud-controller.kubeconfig")
|
||||
|
||||
runtime.ClientAdminCert = path.Join(config.DataDir, "tls", "client-admin.crt")
|
||||
runtime.ClientAdminKey = path.Join(config.DataDir, "tls", "client-admin.key")
|
||||
runtime.ClientControllerCert = path.Join(config.DataDir, "tls", "client-controller.crt")
|
||||
runtime.ClientControllerKey = path.Join(config.DataDir, "tls", "client-controller.key")
|
||||
runtime.ClientCloudControllerCert = path.Join(config.DataDir, "tls", "client-cloud-controller.crt")
|
||||
runtime.ClientCloudControllerKey = path.Join(config.DataDir, "tls", "client-cloud-controller.key")
|
||||
runtime.ClientSchedulerCert = path.Join(config.DataDir, "tls", "client-scheduler.crt")
|
||||
runtime.ClientSchedulerKey = path.Join(config.DataDir, "tls", "client-scheduler.key")
|
||||
runtime.ClientKubeAPICert = path.Join(config.DataDir, "tls", "client-kube-apiserver.crt")
|
||||
@ -573,6 +586,16 @@ func genClientCerts(config *config.Control, runtime *config.ControlRuntime) erro
|
||||
return err
|
||||
}
|
||||
|
||||
certGen, err = factory("cloud-controller-manager", nil, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if certGen {
|
||||
if err := KubeConfig(runtime.KubeConfigCloudController, apiEndpoint, runtime.ServerCA, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -798,3 +821,52 @@ func expired(certFile string) bool {
|
||||
}
|
||||
return certutil.IsCertExpired(certificates[0])
|
||||
}
|
||||
|
||||
func cloudControllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
|
||||
argsMap := map[string]string{
|
||||
"kubeconfig": runtime.KubeConfigCloudController,
|
||||
"allocate-node-cidrs": "true",
|
||||
"cluster-cidr": cfg.ClusterIPRange.String(),
|
||||
"bind-address": localhostIP.String(),
|
||||
"secure-port": "0",
|
||||
"cloud-provider": "k3s",
|
||||
"allow-untagged-cloud": "true",
|
||||
"node-status-update-frequency": "1m",
|
||||
}
|
||||
if cfg.NoLeaderElect {
|
||||
argsMap["leader-elect"] = "false"
|
||||
}
|
||||
|
||||
args := config.GetArgsList(argsMap, cfg.ExtraCloudControllerArgs)
|
||||
|
||||
command := ccmapp.NewCloudControllerManagerCommand()
|
||||
command.SetArgs(args)
|
||||
// register k3s cloud provider
|
||||
|
||||
go func() {
|
||||
for {
|
||||
// check for the cloud controller rbac binding
|
||||
if err := checkForCloudControllerPrivileges(runtime); err != nil {
|
||||
logrus.Infof("Waiting for cloudcontroller rbac role to be created")
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args))
|
||||
logrus.Fatalf("cloud-controller-manager exited: %v", command.Execute())
|
||||
}()
|
||||
}
|
||||
|
||||
func checkForCloudControllerPrivileges(runtime *config.ControlRuntime) error {
|
||||
restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
crb := rbac.NewFactoryFromConfigOrDie(restConfig).Rbac().V1().ClusterRoleBinding()
|
||||
_, err = crb.Get("cloud-controller-manager", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
// Code generated for package deploy by go-bindata DO NOT EDIT. (@generated)
|
||||
// sources:
|
||||
// manifests/ccm.yaml
|
||||
// manifests/coredns.yaml
|
||||
// manifests/local-storage.yaml
|
||||
// manifests/rolebindings.yaml
|
||||
@ -80,6 +81,26 @@ func (fi bindataFileInfo) Sys() interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ccmYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x93\x3f\x4f\x33\x31\x0c\xc6\xf7\x7c\x0a\xab\x4b\xa5\x57\xba\x56\xef\x86\x6e\x84\x81\xbd\x12\xec\xbe\xc4\xb4\xa1\xb9\x38\xb2\x9d\xab\xe0\xd3\xa3\x6b\xcb\xc0\x15\xfa\x6f\xb3\xac\x3c\x3f\x3f\x8e\x6d\x2c\xf1\x95\x44\x23\xe7\x16\xa4\x43\xbf\xc0\x6a\x1b\x96\xf8\x89\x16\x39\x2f\xb6\x0f\xba\x88\xbc\x1c\xfe\xbb\x6d\xcc\xa1\x85\xa7\x54\xd5\x48\x56\x9c\xc8\xf5\x64\x18\xd0\xb0\x75\x00\x19\x7b\x6a\xc1\x27\xae\xa1\xf1\x9c\x4d\x38\x25\x92\xa6\xc7\x8c\x6b\x12\x27\x35\x91\xb6\xae\x01\x2c\xf1\x59\xb8\x16\x1d\x45\x0d\xcc\x66\x0e\x40\x48\xb9\x8a\xa7\x63\x8e\x06\xca\xa6\x0e\x60\x20\xe9\x8e\x39\x2f\x84\x46\xfb\xb0\xa0\xf9\xcd\x3e\xaa\x25\x8c\xc9\xab\xa0\x99\x03\x4d\x98\xf3\x7f\xf3\x1b\xb4\x4b\x35\xb4\x3a\x41\x1c\xbc\x5c\x05\x51\x92\x21\xfa\xa9\x87\x14\xd5\x7e\xef\x6a\x0c\x77\x37\xe3\xd1\x7b\xae\x7f\xfd\xde\x55\xa0\x32\x2e\x83\x1a\x65\x1b\x38\xd5\x7e\x6a\x78\x4d\xf6\xd3\xf8\x7d\x76\x29\x87\xc2\xf1\xdc\x98\x4f\x0a\xed\x4e\xe6\xde\x34\xee\xfe\xed\x7d\x8c\x39\xc4\xbc\xbe\x69\x89\x39\xd1\x8a\xde\xc6\x97\xdf\x2d\x9e\xa9\xea\x00\x4e\x4f\xe6\x62\x0d\xad\xdd\x3b\x79\xdb\xdf\xca\x41\xfe\xa2\x24\x97\x75\x87\x07\x5a\xd0\x53\x0b\xdb\xda\x51\xa3\x1f\x6a\xd4\xbb\xaf\x00\x00\x00\xff\xff\xed\x32\x7e\x6b\xe0\x03\x00\x00")
|
||||
|
||||
func ccmYamlBytes() ([]byte, error) {
|
||||
return bindataRead(
|
||||
_ccmYaml,
|
||||
"ccm.yaml",
|
||||
)
|
||||
}
|
||||
|
||||
func ccmYaml() (*asset, error) {
|
||||
bytes, err := ccmYamlBytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "ccm.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
|
||||
a := &asset{bytes: bytes, info: info}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
var _corednsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\x5f\x6f\xdb\x36\x10\x7f\xf7\xa7\x20\x34\xf4\x6d\x72\x62\x04\xed\x32\xbe\xb5\x71\xd6\x06\x68\x5c\x23\x4e\xfa\x32\x0c\xc5\x99\x3a\x5b\x5c\x28\x1e\x47\x9e\xdc\x78\x5d\xbe\xfb\xa0\xbf\x16\x1d\xa5\x68\x8b\xea\x89\xe2\xf1\x7e\x77\xbc\x3f\xbf\x23\x38\xfd\x11\x7d\xd0\x64\xa5\xd8\xcd\x26\xf7\xda\x66\x52\xac\xd0\xef\xb4\xc2\xd7\x4a\x51\x69\x79\x52\x20\x43\x06\x0c\x72\x22\x84\x85\x02\xa5\x50\xe4\x31\xb3\xa1\xfd\x0f\x0e\x14\x4a\x71\x5f\xae\x31\x0d\xfb\xc0\x58\x4c\xd2\x34\x9d\x0c\xa1\xfd\x1a\xd4\x14\x4a\xce\xc9\xeb\x7f\x81\x35\xd9\xe9\xfd\x79\x98\x6a\x3a\xd9\xcd\xd6\xc8\xd0\x59\xbe\x30\x65\x60\xf4\x37\x64\x30\x32\x6b\x60\x8d\x26\x54\x2b\x51\xdb\xf1\x16\x19\x6b\xfd\x35\x11\x07\xf6\xe0\x9c\xb6\xdb\xc6\x50\x9a\xe1\x06\x4a\xc3\xa1\xf7\xb7\xf1\x4a\x76\x6e\xfb\xd2\x60\x90\x93\x54\x80\xd3\x6f\x3d\x95\xae\x46\x4e\x45\x92\x4c\x84\xf0\x18\xa8\xf4\x0a\xdb\x3d\xb4\x99\x23\x6d\x6b\xb0\x54\x84\x26\x32\xcd\x8f\xa3\xac\x59\xf4\x41\xa8\x7e\x77\xe8\xd7\xad\xae\xd1\x81\xeb\xc5\x67\x60\x95\x7f\x9b\x3d\x4b\xd9\x31\xcc\x16\xf9\x67\x04\xf4\x8d\xb6\x99\xb6\xdb\x28\xae\x60\x2d\x71\xad\xde\x06\x77\x0c\x37\x8a\x37\x94\x4c\xa5\xcb\x80\x51\x8a\x84\x7d\x89\xc9\xcf\x4f\x0f\x19\xbc\xc1\x4d\xed\x5f\x1b\xb0\xaf\x5c\x78\x22\xc4\xd3\xda\x79\x06\x39\x94\xeb\xbf\x51\x71\x9d\xfb\xd1\x52\xff\xe1\x02\xef\x7b\xe7\x82\xec\x46\x6f\xaf\xc1\xfd\x48\xdb\x74\xc7\x2f\xc8\xe3\x46\x1b\x94\xe2\xbf\x3a\xa6\x53\xf9\xf2\x4c\x7c\xa9\x97\xd5\x87\xde\x93\x0f\xfd\x6f\x8e\x60\x38\xef\x7f\x0f\x09\x10\x2f\xbe\x5c\xbc\xbf\x5b\xdd\x5e\xde\x7c\x9a\x7f\xb8\x7e\x7d\xb5\x78\x7c\x21\xb4\x4d\x21\xcb\xfc\x14\xbc\x03\xa1\xdd\xab\x66\x71\xc0\x16\x75\x59\x0b\x6d\x03\xaa\xd2\xe3\x60\xbf\x74\x81\x3d\x42\x31\xd8\xda\x80\x31\x9c\x7b\x2a\xb7\xf9\x38\x70\x7f\xf6\xf1\xe0\x2d\x05\x0e\xe2\x04\x59\x9d\xb4\xf1\x38\x59\x50\x86\xef\xea\xed\xa1\x1f\x1e\x0d\x41\x26\x66\x61\xdc\xe0\x08\xb4\xf3\x54\x20\xe7\x58\x06\x21\x7f\x9f\xbd\x3c\xeb\x05\x1b\xf2\x9f\xc1\x67\x62\xda\xd8\xad\x5a\xce\xec\xa6\x8a\xec\xa6\x3f\xa2\x40\xe5\x28\xce\x4e\xfb\x0d\x43\xe4\x26\xb1\x2f\x03\x19\x64\x6b\x30\x60\x55\x13\xa0\xc7\x27\xf5\x00\xce\x85\x93\xbe\x28\xe6\xe8\x0c\xed\x0b\xfc\x31\x32\x3d\xea\xaf\xf3\x90\x82\x73\xed\x91\x46\xf1\xb8\xeb\x1a\xe0\xa4\x2a\xa3\xf9\x62\x95\x4c\x82\x43\x55\x69\xff\xe2\xd1\x19\xad\x20\x48\x31\x9b\x08\x51\x35\x26\xe3\x76\xdf\x00\xf3\xde\xa1\x14\x37\x64\x8c\xb6\xdb\xbb\xba\xc5\x1b\x4a\x18\xee\xc8\x36\x06\x05\x3c\xdc\x59\xd8\x81\x36\xb0\xae\xea\xb4\x86\x43\x83\x8a\xc9\x37\x67\x8a\x8a\xf3\xde\x0f\x1c\x1f\x77\x9d\xb1\x70\xa6\x07\x1e\x46\xa7\x0e\x74\xa4\xff\xdc\xe5\xbb\xeb\xd5\xeb\xa8\xa1\x17\x47\x11\xae\xef\x49\x06\xfd\x90\xf3\xaa\x2f\x15\xf7\xb8\xaf\x42\xe6\x35\x6b\x05\xe6\x75\x96\x91\x0d\x1f\xac\xd9\x27\x83\xf2\x23\x57\x69\x92\x97\x22\xb9\x7c\xd0\x81\x43\x27\xac\x58\x7b\x15\x5d\xbf\xfa\x2a\x22\x3e\xa2\x4f\x0a\x52\x18\x6d\xcb\x87\xf6\x90\x22\xcb\xa0\x2d\xfa\xde\x97\xf4\x49\x59\x34\x9f\x2e\x60\x7b\xd8\xee\x7a\x47\xce\xa6\xaf\xa6\x67\xf1\xa1\x65\x69\xcc\x92\x8c\x56\x7b\x29\xae\x36\x0b\xe2\xa5\xc7\x80\x35\xbb\x75\xd5\x3c\x18\x39\x7d\x4d\xeb\x42\x73\xb4\x53\xa5\xa3\x20\xbf\x97\x62\xf6\xdb\xe9\xb5\x8e\x5a\xf3\x9f\x12\xc3\xf1\x69\xe5\x4a\x29\x66\xa7\xa7\xc5\x28\x46\x04\x01\x7e\x1b\xa4\xf8\x53\x24\x69\xd5\x84\xc9\xaf\x22\x89\x08\xa1\xe3\xbf\x44\xfc\xd5\xab\xec\xc8\x94\x05\x5e\x57\x59\x8d\xf2\xd6\x45\xab\xa2\xdd\xb4\x39\x34\xb0\x5f\x54\xe7\x97\xc0\xb9\x8c\x28\x27\xba\x0b\x64\x55\x9e\xa5\xa8\xa6\xd9\x81\x4b\xc8\xc7\x76\xfa\x4c\x2d\xc9\xb3\x14\x03\x76\xe9\x1a\x39\xc6\x75\x9e\x98\x14\x19\x29\xee\xe6\xcb\xef\xc5\x49\x59\xb9\x51\xac\xdb\x8b\xaf\x60\x45\x9c\xd7\xa1\x15\xc8\x5e\xab\x71\xcf\x86\x68\x35\xdf\x6b\xde\x5f\x90\x65\x7c\xe0\x61\x6a\xc1\x18\xfa\xbc\xf4\x7a\xa7\x0d\x6e\xf1\x32\x28\x30\x75\xff\xc8\x8a\x8f\xc3\x30\xdc\x0a\x1c\xac\xb5\xd1\xac\xf1\xa8\x38\x20\xcb\xe2\x8d\x54\x2c\x2e\x6f\x3f\xbd\xb9\x5a\xcc\x3f\xad\x2e\x6f\x3e\x5e\x5d\x5c\x46\xe2\xcc\x93\x3b\x56\x00\x63\x46\x12\x77\x43\xc4\x7f\x68\x83\xed\xac\x8f\xd3\x68\xf4\x0e\x2d\x86\xb0\xf4\xb4\xc6\x21\x5e\xce\xec\xde\x22\xc7\x26\x5c\x53\x28\x47\x03\x55\xb4\xe5\x20\xc5\xf9\xe9\xf9\x69\xb4\x1d\x54\x8e\x55\x90\xdf\xdd\xde\x2e\x07\x02\x6d\x35\x6b\x30\x73\x34\xb0\x5f\xa1\x22\x9b\x05\x29\x5e\x0d\x55\x59\x17\x48\x25\xf7\xc2\x97\x03\x59\x28\x95\xc2\x10\x6e\x73\x8f\x21\x27\x93\x35\xec\xda\x7d\x1b\xd0\xa6\xf4\x38\x90\x76\xba\x99\x0d\x5d\xdb\xcf\x9b\x27\x56\x2b\x68\xba\xe2\x3b\xba\x46\x75\x8f\x98\x38\x3c\xe3\xc4\x54\x5f\x98\xb1\x08\xc7\xe9\xaa\x19\xb5\x6b\xe5\x48\xd6\x45\x7a\x54\xd8\x2a\xf6\x8f\x82\x51\xcd\x83\xf4\xd9\x97\x58\xfb\xb4\x1b\x99\xb8\x83\xe1\xf1\xec\xc8\x7d\xf2\x32\x3e\x3c\x2e\x2a\x1e\x6f\xea\x21\xa9\x3a\x2e\x19\x11\x07\xe5\xc1\x3d\xfb\x42\xfe\x86\x09\xae\x9a\xc7\x6c\xda\x8e\xb3\x01\xd2\xb7\xce\xfa\x78\x1a\x8f\xd9\x6c\x6d\x5c\x2d\xe5\xf0\xa1\xb8\x58\x3d\xbe\x98\x0c\xf8\x2f\x3d\x62\x37\x37\xa4\xad\x63\x92\x4b\x47\x28\xec\x19\x85\x86\x7b\xd2\x11\x96\x72\x31\x99\xc5\x2a\xff\x07\x00\x00\xff\xff\xe6\x56\x5e\x9e\xb1\x0e\x00\x00")
|
||||
|
||||
func corednsYamlBytes() ([]byte, error) {
|
||||
@ -212,6 +233,7 @@ func AssetNames() []string {
|
||||
|
||||
// _bindata is a table, holding each asset generator, mapped to its name.
|
||||
var _bindata = map[string]func() (*asset, error){
|
||||
"ccm.yaml": ccmYaml,
|
||||
"coredns.yaml": corednsYaml,
|
||||
"local-storage.yaml": localStorageYaml,
|
||||
"rolebindings.yaml": rolebindingsYaml,
|
||||
@ -259,6 +281,7 @@ type bintree struct {
|
||||
}
|
||||
|
||||
var _bintree = &bintree{nil, map[string]*bintree{
|
||||
"ccm.yaml": &bintree{ccmYaml, map[string]*bintree{}},
|
||||
"coredns.yaml": &bintree{corednsYaml, map[string]*bintree{}},
|
||||
"local-storage.yaml": &bintree{localStorageYaml, map[string]*bintree{}},
|
||||
"rolebindings.yaml": &bintree{rolebindingsYaml, map[string]*bintree{}},
|
||||
|
||||
@ -221,10 +221,17 @@ func (h *handler) podIPs(pods []*core.Pod) ([]string, error) {
|
||||
}
|
||||
|
||||
for _, addr := range node.Status.Addresses {
|
||||
if addr.Type == core.NodeInternalIP {
|
||||
if addr.Type == core.NodeExternalIP {
|
||||
ips[addr.Address] = true
|
||||
}
|
||||
}
|
||||
if len(ips) == 0 {
|
||||
for _, addr := range node.Status.Addresses {
|
||||
if addr.Type == core.NodeInternalIP {
|
||||
ips[addr.Address] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var ipList []string
|
||||
|
||||
Loading…
Reference in New Issue
Block a user