Merge pull request #846 from galal-hussein/k3s_cloud_provider

K3s cloud provider
2024-06-07 19:41:36 +00:00 · 2019-10-16 15:00:22 -07:00 · 2019-10-16 15:00:22 -07:00 · 756a7f1069
commit 756a7f1069
parent c5569cc518 d2c1f66496
38 changed files with 2170 additions and 100 deletions
--- a/go.mod
+++ b/go.mod
@ -117,6 +117,7 @@ require (
 	k8s.io/apimachinery v0.0.0
 	k8s.io/apiserver v0.0.0
 	k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
+	k8s.io/cloud-provider v0.0.0
 	k8s.io/component-base v0.0.0
 	k8s.io/cri-api v0.0.0
 	k8s.io/klog v0.4.0
--- a/go.sum
+++ b/go.sum
@ -563,8 +563,6 @@ github.com/rancher/containerd v1.3.0-k3s.1 h1:8dz25shb4egTLl0nOXQdtllx20LEXsuOs4
 github.com/rancher/containerd v1.3.0-k3s.1/go.mod h1:ZMfzmqce2Z+QSEqdHMfeJs1TZ/UeJ1aDrazjpQT4ehM=
 github.com/rancher/cri-tools v1.16.0-k3s.1 h1:cv/iVFkfvDLfpSqGFwgyQbMKLGRzcXo8AALUsd8s5qE=
 github.com/rancher/cri-tools v1.16.0-k3s.1/go.mod h1:TEKhKv2EJIZp+p9jnEy4C63g8CosJzsI4kyKKkHag+8=
-github.com/rancher/dynamiclistener v0.0.0-20190926205813-e2ac9e2c193d h1:BPBA86IVuNX/S2g7Kry+jXcuftCWPRLpad1iyd1nP20=
-github.com/rancher/dynamiclistener v0.0.0-20190926205813-e2ac9e2c193d/go.mod h1:vNqDnGTkrX8tiBUevkvjZoQoiZOjaahVbMaYRIl25l0=
 github.com/rancher/dynamiclistener v0.1.1-0.20191010011134-8a2488bc860a h1:1bUYAv5U/Ky4YJ9o8gWxX+vNcjpIL3JWNBao70OlkFE=
 github.com/rancher/dynamiclistener v0.1.1-0.20191010011134-8a2488bc860a/go.mod h1:8hbGf35mB7ormKEFqsAgjgeI5rLbj5N764jG41dNhps=
 github.com/rancher/flannel v0.11.0-k3s.1 h1:mIwnfWDafjzQgFkZeJ1AkFrrAT3EdBaA1giE0eLJKo8=
--- a/manifests/ccm.yaml
+++ b/manifests/ccm.yaml
@ -0,0 +1,72 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cloud-controller-manager
+subjects:
+- kind: User
+  name: cloud-controller-manager
+  namespace: kube-system
--- a/pkg/agent/config/config.go
+++ b/pkg/agent/config/config.go
@ -337,6 +337,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
 	nodeConfig.AgentConfig.NodeIP = nodeIP
 	nodeConfig.AgentConfig.NodeName = nodeName
+	nodeConfig.AgentConfig.NodeExternalIP = envInfo.NodeExternalIP
 	nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
 	nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey
 	nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
@ -398,6 +399,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.AgentConfig.NodeTaints = envInfo.Taints
 	nodeConfig.AgentConfig.NodeLabels = envInfo.Labels
 	nodeConfig.AgentConfig.PrivateRegistry = envInfo.PrivateRegistry
+	nodeConfig.AgentConfig.DisableCCM = controlConfig.DisableCCM

 	return nodeConfig, nil
 }
--- a/pkg/agent/run.go
+++ b/pkg/agent/run.go
@ -18,8 +18,19 @@ import (
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/agent"
+	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/rootless"
+	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
+	corev1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+	"github.com/rancher/wrangler/pkg/start"
 	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+	InternalIPLabel = "k3s.io/internal-ip"
+	ExternalIPLabel = "k3s.io/external-ip"
+	HostnameLabel   = "k3s.io/hostname"
 )

 func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) error {
@ -58,6 +69,12 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
 		}
 	}

+	if !nodeConfig.AgentConfig.DisableCCM {
+		if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig); err != nil {
+			return err
+		}
+	}
+
 	<-ctx.Done()
 	return ctx.Err()
 }
@ -124,3 +141,74 @@ func validate() error {

 	return nil
 }
+
+func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent) error {
+	for {
+		nodeController, nodeCache, err := startNodeController(ctx, agentConfig)
+		if err != nil {
+			logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		nodeCached, err := nodeCache.Get(agentConfig.NodeName)
+		if err != nil {
+			logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		node := nodeCached.DeepCopy()
+		updated := updateLabelMap(ctx, agentConfig, node.Labels)
+		if updated {
+			_, err = nodeController.Update(node)
+			if err == nil {
+				logrus.Infof("addresses labels has been set succesfully on node: %s", agentConfig.NodeName)
+				break
+			}
+			logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		logrus.Infof("addresses labels has already been set succesfully on node: %s", agentConfig.NodeName)
+		return nil
+	}
+	return nil
+}
+
+func startNodeController(ctx context.Context, agentConfig *daemonconfig.Agent) (corev1.NodeController, corev1.NodeCache, error) {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", agentConfig.KubeConfigKubelet)
+	if err != nil {
+		return nil, nil, err
+	}
+	coreFactory := core.NewFactoryFromConfigOrDie(restConfig)
+	nodeController := coreFactory.Core().V1().Node()
+	nodeCache := nodeController.Cache()
+	if err := start.All(ctx, 1, coreFactory); err != nil {
+		return nil, nil, err
+	}
+
+	return nodeController, nodeCache, nil
+}
+
+func updateLabelMap(ctx context.Context, agentConfig *daemonconfig.Agent, nodeLabels map[string]string) bool {
+	if nodeLabels == nil {
+		nodeLabels = make(map[string]string)
+	}
+	updated := false
+	if internalIPLabel, ok := nodeLabels[InternalIPLabel]; !ok || internalIPLabel != agentConfig.NodeIP {
+		nodeLabels[InternalIPLabel] = agentConfig.NodeIP
+		updated = true
+	}
+	if hostnameLabel, ok := nodeLabels[HostnameLabel]; !ok || hostnameLabel != agentConfig.NodeName {
+		nodeLabels[HostnameLabel] = agentConfig.NodeName
+		updated = true
+	}
+	nodeExternalIP := agentConfig.NodeExternalIP
+	if externalIPLabel := nodeLabels[ExternalIPLabel]; externalIPLabel != nodeExternalIP && nodeExternalIP != "" {
+		nodeLabels[ExternalIPLabel] = nodeExternalIP
+		updated = true
+	} else if nodeExternalIP == "" && externalIPLabel != "" {
+		delete(nodeLabels, ExternalIPLabel)
+		updated = true
+	}
+	return updated
+}
--- a/pkg/cli/cmds/agent.go
+++ b/pkg/cli/cmds/agent.go
@ -15,6 +15,7 @@ type Agent struct {
 	ResolvConf               string
 	DataDir                  string
 	NodeIP                   string
+	NodeExternalIP           string
 	NodeName                 string
 	ClusterSecret            string
 	PauseImage               string
@ -45,6 +46,11 @@ var (
 		Usage:       "(agent) IP address to advertise for node",
 		Destination: &AgentConfig.NodeIP,
 	}
+	NodeExternalIPFlag = cli.StringFlag{
+		Name:        "node-external-ip",
+		Usage:       "(agent) External IP address to advertise for node",
+		Destination: &AgentConfig.NodeExternalIP,
+	}
 	NodeNameFlag = cli.StringFlag{
 		Name:        "node-name",
 		Usage:       "(agent) Node name",
@ -175,6 +181,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			NodeLabels,
 			NodeTaints,
 			PrivateRegistryFlag,
+			NodeExternalIPFlag,
 		},
 	}
 }
--- a/pkg/cli/cmds/server.go
+++ b/pkg/cli/cmds/server.go
@ -8,33 +8,35 @@ import (
 )

 type Server struct {
-	ClusterCIDR             string
-	ClusterSecret           string
-	ServiceCIDR             string
-	ClusterDNS              string
-	ClusterDomain           string
-	HTTPSPort               int
-	HTTPPort                int
-	DataDir                 string
-	DisableAgent            bool
-	KubeConfigOutput        string
-	KubeConfigMode          string
-	TLSSan                  cli.StringSlice
-	BindAddress             string
-	ExtraAPIArgs            cli.StringSlice
-	ExtraSchedulerArgs      cli.StringSlice
-	ExtraControllerArgs     cli.StringSlice
-	Rootless                bool
-	StoreBootstrap          bool
-	StorageEndpoint         string
-	StorageCAFile           string
-	StorageCertFile         string
-	StorageKeyFile          string
-	AdvertiseIP             string
-	AdvertisePort           int
-	DisableScheduler        bool
-	FlannelBackend          string
-	DefaultLocalStoragePath string
+	ClusterCIDR              string
+	ClusterSecret            string
+	ServiceCIDR              string
+	ClusterDNS               string
+	ClusterDomain            string
+	HTTPSPort                int
+	HTTPPort                 int
+	DataDir                  string
+	DisableAgent             bool
+	KubeConfigOutput         string
+	KubeConfigMode           string
+	TLSSan                   cli.StringSlice
+	BindAddress              string
+	ExtraAPIArgs             cli.StringSlice
+	ExtraSchedulerArgs       cli.StringSlice
+	ExtraControllerArgs      cli.StringSlice
+	ExtraCloudControllerArgs cli.StringSlice
+	Rootless                 bool
+	StoreBootstrap           bool
+	StorageEndpoint          string
+	StorageCAFile            string
+	StorageCertFile          string
+	StorageKeyFile           string
+	AdvertiseIP              string
+	AdvertisePort            int
+	DisableScheduler         bool
+	FlannelBackend           string
+	DefaultLocalStoragePath  string
+	DisableCCM               bool
 }

 var ServerConfig Server
@ -143,6 +145,11 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage: "Customized flag for kube-controller-manager process",
 				Value: &ServerConfig.ExtraControllerArgs,
 			},
+			cli.StringSliceFlag{
+				Name:  "kube-cloud-controller-arg",
+				Usage: "Customized flag for kube-cloud-controller-manager process",
+				Value: &ServerConfig.ExtraCloudControllerArgs,
+			},
 			cli.BoolFlag{
 				Name:        "rootless",
 				Usage:       "(experimental) Run rootless",
@ -194,6 +201,11 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage:       "Disable Kubernetes default scheduler",
 				Destination: &ServerConfig.DisableScheduler,
 			},
+			cli.BoolFlag{
+				Name:        "disable-cloud-controller",
+				Usage:       "Disable k3s default cloud controller manager",
+				Destination: &ServerConfig.DisableCCM,
+			},
 			cli.StringFlag{
 				Name:        "flannel-backend",
 				Usage:       fmt.Sprintf("(experimental) One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard),
@ -219,6 +231,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 			NodeLabels,
 			NodeTaints,
 			PrivateRegistryFlag,
+			NodeExternalIPFlag,
 		},
 	}
 }
--- a/pkg/cli/server/server.go
+++ b/pkg/cli/server/server.go
@ -85,6 +85,8 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
 	serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
 	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
+	serverConfig.ControlConfig.ExtraCloudControllerArgs = cfg.ExtraCloudControllerArgs
+	serverConfig.ControlConfig.DisableCCM = cfg.DisableCCM

 	if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
 		cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)
--- a/pkg/cloudprovider/cloudprovider.go
+++ b/pkg/cloudprovider/cloudprovider.go
@ -0,0 +1,57 @@
+package cloudprovider
+
+import (
+	"context"
+	"io"
+
+	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
+	coreclient "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+	"github.com/rancher/wrangler/pkg/start"
+	cloudprovider "k8s.io/cloud-provider"
+)
+
+type k3s struct {
+	NodeCache coreclient.NodeCache
+}
+
+func init() {
+	cloudprovider.RegisterCloudProvider("k3s", func(config io.Reader) (cloudprovider.Interface, error) {
+		return &k3s{}, nil
+	})
+}
+
+func (k *k3s) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) {
+	coreFactory := core.NewFactoryFromConfigOrDie(clientBuilder.ConfigOrDie("cloud-controller-manager"))
+
+	go start.All(context.Background(), 1, coreFactory)
+
+	k.NodeCache = coreFactory.Core().V1().Node().Cache()
+}
+
+func (k *k3s) Instances() (cloudprovider.Instances, bool) {
+	return k, true
+}
+
+func (k *k3s) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
+	return nil, false
+}
+
+func (k *k3s) Zones() (cloudprovider.Zones, bool) {
+	return nil, false
+}
+
+func (k *k3s) Clusters() (cloudprovider.Clusters, bool) {
+	return nil, false
+}
+
+func (k *k3s) Routes() (cloudprovider.Routes, bool) {
+	return nil, false
+}
+
+func (k *k3s) ProviderName() string {
+	return "k3s"
+}
+
+func (k *k3s) HasClusterID() bool {
+	return false
+}
--- a/pkg/cloudprovider/instances.go
+++ b/pkg/cloudprovider/instances.go
@ -0,0 +1,85 @@
+package cloudprovider
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	cloudprovider "k8s.io/cloud-provider"
+)
+
+const (
+	InternalIPLabel = "k3s.io/internal-ip"
+	ExternalIPLabel = "k3s.io/external-ip"
+	HostnameLabel   = "k3s.io/hostname"
+)
+
+func (k *k3s) AddSSHKeyToAllInstances(ctx context.Context, user string, keyData []byte) error {
+	return cloudprovider.NotImplemented
+}
+
+func (k *k3s) CurrentNodeName(ctx context.Context, hostname string) (types.NodeName, error) {
+	return types.NodeName(hostname), nil
+}
+
+func (k *k3s) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) {
+	return true, nil
+}
+
+func (k *k3s) InstanceID(ctx context.Context, nodeName types.NodeName) (string, error) {
+	_, err := k.NodeCache.Get(string(nodeName))
+	if err != nil {
+		return "", fmt.Errorf("Failed to find node %s: %v", nodeName, err)
+	}
+	return string(nodeName), nil
+}
+
+func (k *k3s) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) {
+	return true, cloudprovider.NotImplemented
+}
+
+func (k *k3s) InstanceType(ctx context.Context, name types.NodeName) (string, error) {
+	_, err := k.InstanceID(ctx, name)
+	if err != nil {
+		return "", err
+	}
+	return "k3s", nil
+}
+
+func (k *k3s) InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) {
+	return "", cloudprovider.NotImplemented
+}
+
+func (k *k3s) NodeAddresses(ctx context.Context, name types.NodeName) ([]corev1.NodeAddress, error) {
+	addresses := []corev1.NodeAddress{}
+	node, err := k.NodeCache.Get(string(name))
+	if err != nil {
+		return nil, fmt.Errorf("Failed to find node %s: %v", name, err)
+	}
+	// check internal address
+	if node.Labels[InternalIPLabel] != "" {
+		addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: node.Labels[InternalIPLabel]})
+	} else {
+		logrus.Infof("couldn't find node internal ip label on node %s", name)
+	}
+
+	// check external address
+	if node.Labels[ExternalIPLabel] != "" {
+		addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: node.Labels[ExternalIPLabel]})
+	}
+
+	// check hostname
+	if node.Labels[HostnameLabel] != "" {
+		addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: node.Labels[HostnameLabel]})
+	} else {
+		logrus.Infof("couldn't find node hostname label on node %s", name)
+	}
+
+	return addresses, nil
+}
+
+func (k *k3s) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]corev1.NodeAddress, error) {
+	return nil, cloudprovider.NotImplemented
+}
--- a/pkg/daemons/agent/agent.go
+++ b/pkg/daemons/agent/agent.go
@ -133,6 +133,10 @@ func kubelet(cfg *config.Agent) {
 	if len(cfg.NodeTaints) > 0 {
 		argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
 	}
+	if !cfg.DisableCCM {
+		argsMap["cloud-provider"] = "external"
+	}
+
 	args := config.GetArgsList(argsMap, cfg.ExtraKubeletArgs)
 	command.SetArgs(args)

--- a/pkg/daemons/config/types.go
+++ b/pkg/daemons/config/types.go
@ -63,6 +63,7 @@ type Agent struct {
 	KubeConfigKubelet   string
 	KubeConfigKubeProxy string
 	NodeIP              string
+	NodeExternalIP      string
 	RuntimeSocket       string
 	ListenAddress       string
 	ClientCA            string
@ -77,33 +78,36 @@ type Agent struct {
 	IPSECPSK            string
 	StrongSwanDir       string
 	PrivateRegistry     string
+	DisableCCM          bool
 }

 type Control struct {
-	AdvertisePort           int
-	AdvertiseIP             string
-	ListenPort              int
-	HTTPSPort               int
-	ClusterSecret           string
-	ClusterIPRange          *net.IPNet
-	ServiceIPRange          *net.IPNet
-	ClusterDNS              net.IP
-	ClusterDomain           string
-	NoCoreDNS               bool
-	KubeConfigOutput        string
-	KubeConfigMode          string
-	DataDir                 string
-	Skips                   []string
-	BootstrapReadOnly       bool
-	Storage                 endpoint.Config
-	NoScheduler             bool
-	ExtraAPIArgs            []string
-	ExtraControllerArgs     []string
-	ExtraSchedulerAPIArgs   []string
-	NoLeaderElect           bool
-	FlannelBackend          string
-	IPSECPSK                string
-	DefaultLocalStoragePath string
+	AdvertisePort            int
+	AdvertiseIP              string
+	ListenPort               int
+	HTTPSPort                int
+	ClusterSecret            string
+	ClusterIPRange           *net.IPNet
+	ServiceIPRange           *net.IPNet
+	ClusterDNS               net.IP
+	ClusterDomain            string
+	NoCoreDNS                bool
+	KubeConfigOutput         string
+	KubeConfigMode           string
+	DataDir                  string
+	Skips                    []string
+	BootstrapReadOnly        bool
+	Storage                  endpoint.Config
+	NoScheduler              bool
+	ExtraAPIArgs             []string
+	ExtraControllerArgs      []string
+	ExtraSchedulerAPIArgs    []string
+	ExtraCloudControllerArgs []string
+	NoLeaderElect            bool
+	FlannelBackend           string
+	IPSECPSK                 string
+	DefaultLocalStoragePath  string
+	DisableCCM               bool

 	Runtime *ControlRuntime `json:"-"`
 }
@ -130,10 +134,11 @@ type ControlRuntime struct {
 	ClientKubeAPIKey  string
 	NodePasswdFile    string

-	KubeConfigAdmin      string
-	KubeConfigController string
-	KubeConfigScheduler  string
-	KubeConfigAPIServer  string
+	KubeConfigAdmin           string
+	KubeConfigController      string
+	KubeConfigScheduler       string
+	KubeConfigAPIServer       string
+	KubeConfigCloudController string

 	ServingKubeAPICert string
 	ServingKubeAPIKey  string
@ -146,13 +151,15 @@ type ControlRuntime struct {
 	ClientAuthProxyCert string
 	ClientAuthProxyKey  string

-	ClientAdminCert      string
-	ClientAdminKey       string
-	ClientControllerCert string
-	ClientControllerKey  string
-	ClientSchedulerCert  string
-	ClientSchedulerKey   string
-	ClientKubeProxyCert  string
+	ClientAdminCert           string
+	ClientAdminKey            string
+	ClientControllerCert      string
+	ClientControllerKey       string
+	ClientSchedulerCert       string
+	ClientSchedulerKey        string
+	ClientKubeProxyCert       string
+	ClientCloudControllerCert string
+	ClientCloudControllerKey  string
 }

 type ArgString []string
--- a/pkg/daemons/control/server.go
+++ b/pkg/daemons/control/server.go
@ -22,11 +22,17 @@ import (
 	"time"

 	certutil "github.com/rancher/dynamiclistener/cert"
+	// registering k3s cloud provider
+	_ "github.com/rancher/k3s/pkg/cloudprovider"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/kine/pkg/client"
 	"github.com/rancher/kine/pkg/endpoint"
+	"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
 	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/client-go/tools/clientcmd"
+	ccmapp "k8s.io/kubernetes/cmd/cloud-controller-manager/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
 	sapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
@ -94,6 +100,10 @@ func Server(ctx context.Context, cfg *config.Control) error {

 	controllerManager(cfg, runtime)

+	if !cfg.DisableCCM {
+		cloudControllerManager(cfg, runtime)
+	}
+
 	return nil
 }

@ -267,11 +277,14 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.KubeConfigController = path.Join(config.DataDir, "cred", "controller.kubeconfig")
 	runtime.KubeConfigScheduler = path.Join(config.DataDir, "cred", "scheduler.kubeconfig")
 	runtime.KubeConfigAPIServer = path.Join(config.DataDir, "cred", "api-server.kubeconfig")
+	runtime.KubeConfigCloudController = path.Join(config.DataDir, "cred", "cloud-controller.kubeconfig")

 	runtime.ClientAdminCert = path.Join(config.DataDir, "tls", "client-admin.crt")
 	runtime.ClientAdminKey = path.Join(config.DataDir, "tls", "client-admin.key")
 	runtime.ClientControllerCert = path.Join(config.DataDir, "tls", "client-controller.crt")
 	runtime.ClientControllerKey = path.Join(config.DataDir, "tls", "client-controller.key")
+	runtime.ClientCloudControllerCert = path.Join(config.DataDir, "tls", "client-cloud-controller.crt")
+	runtime.ClientCloudControllerKey = path.Join(config.DataDir, "tls", "client-cloud-controller.key")
 	runtime.ClientSchedulerCert = path.Join(config.DataDir, "tls", "client-scheduler.crt")
 	runtime.ClientSchedulerKey = path.Join(config.DataDir, "tls", "client-scheduler.key")
 	runtime.ClientKubeAPICert = path.Join(config.DataDir, "tls", "client-kube-apiserver.crt")
@ -573,6 +586,16 @@ func genClientCerts(config *config.Control, runtime *config.ControlRuntime) erro
 		return err
 	}

+	certGen, err = factory("cloud-controller-manager", nil, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey)
+	if err != nil {
+		return err
+	}
+	if certGen {
+		if err := KubeConfig(runtime.KubeConfigCloudController, apiEndpoint, runtime.ServerCA, runtime.ClientCloudControllerCert, runtime.ClientCloudControllerKey); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

@ -798,3 +821,52 @@ func expired(certFile string) bool {
 	}
 	return certutil.IsCertExpired(certificates[0])
 }
+
+func cloudControllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
+	argsMap := map[string]string{
+		"kubeconfig":                   runtime.KubeConfigCloudController,
+		"allocate-node-cidrs":          "true",
+		"cluster-cidr":                 cfg.ClusterIPRange.String(),
+		"bind-address":                 localhostIP.String(),
+		"secure-port":                  "0",
+		"cloud-provider":               "k3s",
+		"allow-untagged-cloud":         "true",
+		"node-status-update-frequency": "1m",
+	}
+	if cfg.NoLeaderElect {
+		argsMap["leader-elect"] = "false"
+	}
+
+	args := config.GetArgsList(argsMap, cfg.ExtraCloudControllerArgs)
+
+	command := ccmapp.NewCloudControllerManagerCommand()
+	command.SetArgs(args)
+	// register k3s cloud provider
+
+	go func() {
+		for {
+			// check for the cloud controller rbac binding
+			if err := checkForCloudControllerPrivileges(runtime); err != nil {
+				logrus.Infof("Waiting for cloudcontroller rbac role to be created")
+				time.Sleep(time.Second)
+				continue
+			}
+			break
+		}
+		logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args))
+		logrus.Fatalf("cloud-controller-manager exited: %v", command.Execute())
+	}()
+}
+
+func checkForCloudControllerPrivileges(runtime *config.ControlRuntime) error {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigAdmin)
+	if err != nil {
+		return err
+	}
+	crb := rbac.NewFactoryFromConfigOrDie(restConfig).Rbac().V1().ClusterRoleBinding()
+	_, err = crb.Get("cloud-controller-manager", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
--- a/pkg/deploy/zz_generated_bindata.go
+++ b/pkg/deploy/zz_generated_bindata.go
@ -1,5 +1,6 @@
 // Code generated for package deploy by go-bindata DO NOT EDIT. (@generated)
 // sources:
+// manifests/ccm.yaml
 // manifests/coredns.yaml
 // manifests/local-storage.yaml
 // manifests/rolebindings.yaml
@ -80,6 +81,26 @@ func (fi bindataFileInfo) Sys() interface{} {
 	return nil
 }

+var _ccmYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x93\x3f\x4f\x33\x31\x0c\xc6\xf7\x7c\x0a\xab\x4b\xa5\x57\xba\x56\xef\x86\x6e\x84\x81\xbd\x12\xec\xbe\xc4\xb4\xa1\xb9\x38\xb2\x9d\xab\xe0\xd3\xa3\x6b\xcb\xc0\x15\xfa\x6f\xb3\xac\x3c\x3f\x3f\x8e\x6d\x2c\xf1\x95\x44\x23\xe7\x16\xa4\x43\xbf\xc0\x6a\x1b\x96\xf8\x89\x16\x39\x2f\xb6\x0f\xba\x88\xbc\x1c\xfe\xbb\x6d\xcc\xa1\x85\xa7\x54\xd5\x48\x56\x9c\xc8\xf5\x64\x18\xd0\xb0\x75\x00\x19\x7b\x6a\xc1\x27\xae\xa1\xf1\x9c\x4d\x38\x25\x92\xa6\xc7\x8c\x6b\x12\x27\x35\x91\xb6\xae\x01\x2c\xf1\x59\xb8\x16\x1d\x45\x0d\xcc\x66\x0e\x40\x48\xb9\x8a\xa7\x63\x8e\x06\xca\xa6\x0e\x60\x20\xe9\x8e\x39\x2f\x84\x46\xfb\xb0\xa0\xf9\xcd\x3e\xaa\x25\x8c\xc9\xab\xa0\x99\x03\x4d\x98\xf3\x7f\xf3\x1b\xb4\x4b\x35\xb4\x3a\x41\x1c\xbc\x5c\x05\x51\x92\x21\xfa\xa9\x87\x14\xd5\x7e\xef\x6a\x0c\x77\x37\xe3\xd1\x7b\xae\x7f\xfd\xde\x55\xa0\x32\x2e\x83\x1a\x65\x1b\x38\xd5\x7e\x6a\x78\x4d\xf6\xd3\xf8\x7d\x76\x29\x87\xc2\xf1\xdc\x98\x4f\x0a\xed\x4e\xe6\xde\x34\xee\xfe\xed\x7d\x8c\x39\xc4\xbc\xbe\x69\x89\x39\xd1\x8a\xde\xc6\x97\xdf\x2d\x9e\xa9\xea\x00\x4e\x4f\xe6\x62\x0d\xad\xdd\x3b\x79\xdb\xdf\xca\x41\xfe\xa2\x24\x97\x75\x87\x07\x5a\xd0\x53\x0b\xdb\xda\x51\xa3\x1f\x6a\xd4\xbb\xaf\x00\x00\x00\xff\xff\xed\x32\x7e\x6b\xe0\x03\x00\x00")
+
+func ccmYamlBytes() ([]byte, error) {
+	return bindataRead(
+		_ccmYaml,
+		"ccm.yaml",
+	)
+}
+
+func ccmYaml() (*asset, error) {
+	bytes, err := ccmYamlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "ccm.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
 var _corednsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\x5f\x6f\xdb\x36\x10\x7f\xf7\xa7\x20\x34\xf4\x6d\x72\x62\x04\xed\x32\xbe\xb5\x71\xd6\x06\x68\x5c\x23\x4e\xfa\x32\x0c\xc5\x99\x3a\x5b\x5c\x28\x1e\x47\x9e\xdc\x78\x5d\xbe\xfb\xa0\xbf\x16\x1d\xa5\x68\x8b\xea\x89\xe2\xf1\x7e\x77\xbc\x3f\xbf\x23\x38\xfd\x11\x7d\xd0\x64\xa5\xd8\xcd\x26\xf7\xda\x66\x52\xac\xd0\xef\xb4\xc2\xd7\x4a\x51\x69\x79\x52\x20\x43\x06\x0c\x72\x22\x84\x85\x02\xa5\x50\xe4\x31\xb3\xa1\xfd\x0f\x0e\x14\x4a\x71\x5f\xae\x31\x0d\xfb\xc0\x58\x4c\xd2\x34\x9d\x0c\xa1\xfd\x1a\xd4\x14\x4a\xce\xc9\xeb\x7f\x81\x35\xd9\xe9\xfd\x79\x98\x6a\x3a\xd9\xcd\xd6\xc8\xd0\x59\xbe\x30\x65\x60\xf4\x37\x64\x30\x32\x6b\x60\x8d\x26\x54\x2b\x51\xdb\xf1\x16\x19\x6b\xfd\x35\x11\x07\xf6\xe0\x9c\xb6\xdb\xc6\x50\x9a\xe1\x06\x4a\xc3\xa1\xf7\xb7\xf1\x4a\x76\x6e\xfb\xd2\x60\x90\x93\x54\x80\xd3\x6f\x3d\x95\xae\x46\x4e\x45\x92\x4c\x84\xf0\x18\xa8\xf4\x0a\xdb\x3d\xb4\x99\x23\x6d\x6b\xb0\x54\x84\x26\x32\xcd\x8f\xa3\xac\x59\xf4\x41\xa8\x7e\x77\xe8\xd7\xad\xae\xd1\x81\xeb\xc5\x67\x60\x95\x7f\x9b\x3d\x4b\xd9\x31\xcc\x16\xf9\x67\x04\xf4\x8d\xb6\x99\xb6\xdb\x28\xae\x60\x2d\x71\xad\xde\x06\x77\x0c\x37\x8a\x37\x94\x4c\xa5\xcb\x80\x51\x8a\x84\x7d\x89\xc9\xcf\x4f\x0f\x19\xbc\xc1\x4d\xed\x5f\x1b\xb0\xaf\x5c\x78\x22\xc4\xd3\xda\x79\x06\x39\x94\xeb\xbf\x51\x71\x9d\xfb\xd1\x52\xff\xe1\x02\xef\x7b\xe7\x82\xec\x46\x6f\xaf\xc1\xfd\x48\xdb\x74\xc7\x2f\xc8\xe3\x46\x1b\x94\xe2\xbf\x3a\xa6\x53\xf9\xf2\x4c\x7c\xa9\x97\xd5\x87\xde\x93\x0f\xfd\x6f\x8e\x60\x38\xef\x7f\x0f\x09\x10\x2f\xbe\x5c\xbc\xbf\x5b\xdd\x5e\xde\x7c\x9a\x7f\xb8\x7e\x7d\xb5\x78\x7c\x21\xb4\x4d\x21\xcb\xfc\x14\xbc\x03\xa1\xdd\xab\x66\x71\xc0\x16\x75\x59\x0b\x6d\x03\xaa\xd2\xe3\x60\xbf\x74\x81\x3d\x42\x31\xd8\xda\x80\x31\x9c\x7b\x2a\xb7\xf9\x38\x70\x7f\xf6\xf1\xe0\x2d\x05\x0e\xe2\x04\x59\x9d\xb4\xf1\x38\x59\x50\x86\xef\xea\xed\xa1\x1f\x1e\x0d\x41\x26\x66\x61\xdc\xe0\x08\xb4\xf3\x54\x20\xe7\x58\x06\x21\x7f\x9f\xbd\x3c\xeb\x05\x1b\xf2\x9f\xc1\x67\x62\xda\xd8\xad\x5a\xce\xec\xa6\x8a\xec\xa6\x3f\xa2\x40\xe5\x28\xce\x4e\xfb\x0d\x43\xe4\x26\xb1\x2f\x03\x19\x64\x6b\x30\x60\x55\x13\xa0\xc7\x27\xf5\x00\xce\x85\x93\xbe\x28\xe6\xe8\x0c\xed\x0b\xfc\x31\x32\x3d\xea\xaf\xf3\x90\x82\x73\xed\x91\x46\xf1\xb8\xeb\x1a\xe0\xa4\x2a\xa3\xf9\x62\x95\x4c\x82\x43\x55\x69\xff\xe2\xd1\x19\xad\x20\x48\x31\x9b\x08\x51\x35\x26\xe3\x76\xdf\x00\xf3\xde\xa1\x14\x37\x64\x8c\xb6\xdb\xbb\xba\xc5\x1b\x4a\x18\xee\xc8\x36\x06\x05\x3c\xdc\x59\xd8\x81\x36\xb0\xae\xea\xb4\x86\x43\x83\x8a\xc9\x37\x67\x8a\x8a\xf3\xde\x0f\x1c\x1f\x77\x9d\xb1\x70\xa6\x07\x1e\x46\xa7\x0e\x74\xa4\xff\xdc\xe5\xbb\xeb\xd5\xeb\xa8\xa1\x17\x47\x11\xae\xef\x49\x06\xfd\x90\xf3\xaa\x2f\x15\xf7\xb8\xaf\x42\xe6\x35\x6b\x05\xe6\x75\x96\x91\x0d\x1f\xac\xd9\x27\x83\xf2\x23\x57\x69\x92\x97\x22\xb9\x7c\xd0\x81\x43\x27\xac\x58\x7b\x15\x5d\xbf\xfa\x2a\x22\x3e\xa2\x4f\x0a\x52\x18\x6d\xcb\x87\xf6\x90\x22\xcb\xa0\x2d\xfa\xde\x97\xf4\x49\x59\x34\x9f\x2e\x60\x7b\xd8\xee\x7a\x47\xce\xa6\xaf\xa6\x67\xf1\xa1\x65\x69\xcc\x92\x8c\x56\x7b\x29\xae\x36\x0b\xe2\xa5\xc7\x80\x35\xbb\x75\xd5\x3c\x18\x39\x7d\x4d\xeb\x42\x73\xb4\x53\xa5\xa3\x20\xbf\x97\x62\xf6\xdb\xe9\xb5\x8e\x5a\xf3\x9f\x12\xc3\xf1\x69\xe5\x4a\x29\x66\xa7\xa7\xc5\x28\x46\x04\x01\x7e\x1b\xa4\xf8\x53\x24\x69\xd5\x84\xc9\xaf\x22\x89\x08\xa1\xe3\xbf\x44\xfc\xd5\xab\xec\xc8\x94\x05\x5e\x57\x59\x8d\xf2\xd6\x45\xab\xa2\xdd\xb4\x39\x34\xb0\x5f\x54\xe7\x97\xc0\xb9\x8c\x28\x27\xba\x0b\x64\x55\x9e\xa5\xa8\xa6\xd9\x81\x4b\xc8\xc7\x76\xfa\x4c\x2d\xc9\xb3\x14\x03\x76\xe9\x1a\x39\xc6\x75\x9e\x98\x14\x19\x29\xee\xe6\xcb\xef\xc5\x49\x59\xb9\x51\xac\xdb\x8b\xaf\x60\x45\x9c\xd7\xa1\x15\xc8\x5e\xab\x71\xcf\x86\x68\x35\xdf\x6b\xde\x5f\x90\x65\x7c\xe0\x61\x6a\xc1\x18\xfa\xbc\xf4\x7a\xa7\x0d\x6e\xf1\x32\x28\x30\x75\xff\xc8\x8a\x8f\xc3\x30\xdc\x0a\x1c\xac\xb5\xd1\xac\xf1\xa8\x38\x20\xcb\xe2\x8d\x54\x2c\x2e\x6f\x3f\xbd\xb9\x5a\xcc\x3f\xad\x2e\x6f\x3e\x5e\x5d\x5c\x46\xe2\xcc\x93\x3b\x56\x00\x63\x46\x12\x77\x43\xc4\x7f\x68\x83\xed\xac\x8f\xd3\x68\xf4\x0e\x2d\x86\xb0\xf4\xb4\xc6\x21\x5e\xce\xec\xde\x22\xc7\x26\x5c\x53\x28\x47\x03\x55\xb4\xe5\x20\xc5\xf9\xe9\xf9\x69\xb4\x1d\x54\x8e\x55\x90\xdf\xdd\xde\x2e\x07\x02\x6d\x35\x6b\x30\x73\x34\xb0\x5f\xa1\x22\x9b\x05\x29\x5e\x0d\x55\x59\x17\x48\x25\xf7\xc2\x97\x03\x59\x28\x95\xc2\x10\x6e\x73\x8f\x21\x27\x93\x35\xec\xda\x7d\x1b\xd0\xa6\xf4\x38\x90\x76\xba\x99\x0d\x5d\xdb\xcf\x9b\x27\x56\x2b\x68\xba\xe2\x3b\xba\x46\x75\x8f\x98\x38\x3c\xe3\xc4\x54\x5f\x98\xb1\x08\xc7\xe9\xaa\x19\xb5\x6b\xe5\x48\xd6\x45\x7a\x54\xd8\x2a\xf6\x8f\x82\x51\xcd\x83\xf4\xd9\x97\x58\xfb\xb4\x1b\x99\xb8\x83\xe1\xf1\xec\xc8\x7d\xf2\x32\x3e\x3c\x2e\x2a\x1e\x6f\xea\x21\xa9\x3a\x2e\x19\x11\x07\xe5\xc1\x3d\xfb\x42\xfe\x86\x09\xae\x9a\xc7\x6c\xda\x8e\xb3\x01\xd2\xb7\xce\xfa\x78\x1a\x8f\xd9\x6c\x6d\x5c\x2d\xe5\xf0\xa1\xb8\x58\x3d\xbe\x98\x0c\xf8\x2f\x3d\x62\x37\x37\xa4\xad\x63\x92\x4b\x47\x28\xec\x19\x85\x86\x7b\xd2\x11\x96\x72\x31\x99\xc5\x2a\xff\x07\x00\x00\xff\xff\xe6\x56\x5e\x9e\xb1\x0e\x00\x00")

 func corednsYamlBytes() ([]byte, error) {
@ -212,6 +233,7 @@ func AssetNames() []string {

 // _bindata is a table, holding each asset generator, mapped to its name.
 var _bindata = map[string]func() (*asset, error){
+	"ccm.yaml":           ccmYaml,
 	"coredns.yaml":       corednsYaml,
 	"local-storage.yaml": localStorageYaml,
 	"rolebindings.yaml":  rolebindingsYaml,
@ -259,6 +281,7 @@ type bintree struct {
 }

 var _bintree = &bintree{nil, map[string]*bintree{
+	"ccm.yaml":           &bintree{ccmYaml, map[string]*bintree{}},
 	"coredns.yaml":       &bintree{corednsYaml, map[string]*bintree{}},
 	"local-storage.yaml": &bintree{localStorageYaml, map[string]*bintree{}},
 	"rolebindings.yaml":  &bintree{rolebindingsYaml, map[string]*bintree{}},
--- a/pkg/servicelb/controller.go
+++ b/pkg/servicelb/controller.go
@ -221,10 +221,17 @@ func (h *handler) podIPs(pods []*core.Pod) ([]string, error) {
 		}

 		for _, addr := range node.Status.Addresses {
-			if addr.Type == core.NodeInternalIP {
+			if addr.Type == core.NodeExternalIP {
 				ips[addr.Address] = true
 			}
 		}
+		if len(ips) == 0 {
+			for _, addr := range node.Status.Addresses {
+				if addr.Type == core.NodeInternalIP {
+					ips[addr.Address] = true
+				}
+			}
+		}
 	}

 	var ipList []string
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/BUILD
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/BUILD
@ -0,0 +1,59 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "controllermanager.go",
+        "core.go",
+    ],
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/cloud-controller-manager/app/config:go_default_library",
+        "//cmd/cloud-controller-manager/app/options:go_default_library",
+        "//cmd/controller-manager/app:go_default_library",
+        "//pkg/controller/cloud:go_default_library",
+        "//pkg/controller/route:go_default_library",
+        "//pkg/controller/service:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/util/configz:go_default_library",
+        "//pkg/util/flag:go_default_library",
+        "//pkg/version/verflag:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/term:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",
+        "//staging/src/k8s.io/cloud-provider:go_default_library",
+        "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
+        "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
+        "//staging/src/k8s.io/component-base/version:go_default_library",
+        "//vendor/github.com/spf13/cobra:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+        "//vendor/k8s.io/utils/net:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//cmd/cloud-controller-manager/app/apis/config:all-srcs",
+        "//cmd/cloud-controller-manager/app/config:all-srcs",
+        "//cmd/cloud-controller-manager/app/options:all-srcs",
+        "//cmd/cloud-controller-manager/app/testing:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/BUILD
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/BUILD
@ -0,0 +1,38 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "register.go",
+        "types.go",
+        "zz_generated.deepcopy.go",
+    ],
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/controller/apis/config:go_default_library",
+        "//pkg/controller/service/config:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//cmd/cloud-controller-manager/app/apis/config/scheme:all-srcs",
+        "//cmd/cloud-controller-manager/app/apis/config/v1alpha1:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/doc.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/doc.go
@ -0,0 +1,20 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:deepcopy-gen=package
+// +groupName=cloudcontrollermanager.config.k8s.io
+
+package config // import "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config"
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/register.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/register.go
@ -0,0 +1,42 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// GroupName is the group name used in this package
+const GroupName = "cloudcontrollermanager.config.k8s.io"
+
+var (
+	// SchemeGroupVersion is group version used to register these objects
+	SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal}
+	// SchemeBuilder is the scheme builder with scheme init functions to run for this API package
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	// AddToScheme is a global function that registers this API group & version to a scheme
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+// addKnownTypes registers known types to the given scheme
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&CloudControllerManagerConfiguration{},
+	)
+	return nil
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme/BUILD
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme/BUILD
@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["scheme.go"],
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/cloud-controller-manager/app/apis/config:go_default_library",
+        "//cmd/cloud-controller-manager/app/apis/config/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme/scheme.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme/scheme.go
@ -0,0 +1,44 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheme
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config"
+	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1"
+)
+
+var (
+	// Scheme defines methods for serializing and deserializing API objects.
+	Scheme = runtime.NewScheme()
+	// Codecs provides methods for retrieving codecs and serializers for specific
+	// versions and content types.
+	Codecs = serializer.NewCodecFactory(Scheme)
+)
+
+func init() {
+	AddToScheme(Scheme)
+}
+
+// AddToScheme adds the types of this group into the given scheme.
+func AddToScheme(scheme *runtime.Scheme) {
+	utilruntime.Must(config.AddToScheme(scheme))
+	utilruntime.Must(v1alpha1.AddToScheme(scheme))
+	utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion))
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/types.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/types.go
@ -0,0 +1,43 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config"
+	serviceconfig "k8s.io/kubernetes/pkg/controller/service/config"
+)
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// CloudControllerManagerConfiguration contains elements describing cloud-controller manager.
+type CloudControllerManagerConfiguration struct {
+	metav1.TypeMeta
+
+	// Generic holds configuration for a generic controller-manager
+	Generic kubectrlmgrconfig.GenericControllerManagerConfiguration
+	// KubeCloudSharedConfiguration holds configuration for shared related features
+	// both in cloud controller manager and kube-controller manager.
+	KubeCloudShared kubectrlmgrconfig.KubeCloudSharedConfiguration
+
+	// ServiceControllerConfiguration holds configuration for ServiceController
+	// related features.
+	ServiceController serviceconfig.ServiceControllerConfiguration
+
+	// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
+	NodeStatusUpdateFrequency metav1.Duration
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/BUILD
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/BUILD
@ -0,0 +1,51 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "defaults.go",
+        "doc.go",
+        "register.go",
+        "types.go",
+        "zz_generated.conversion.go",
+        "zz_generated.deepcopy.go",
+        "zz_generated.defaults.go",
+    ],
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/cloud-controller-manager/app/apis/config:go_default_library",
+        "//pkg/controller/apis/config/v1alpha1:go_default_library",
+        "//pkg/controller/service/config/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/kube-controller-manager/config/v1alpha1:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["defaults_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/defaults.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/defaults.go
@ -0,0 +1,51 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	kubectrlmgrconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/apis/config/v1alpha1"
+	serviceconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/service/config/v1alpha1"
+)
+
+func addDefaultingFuncs(scheme *runtime.Scheme) error {
+	return RegisterDefaults(scheme)
+}
+
+func SetDefaults_CloudControllerManagerConfiguration(obj *CloudControllerManagerConfiguration) {
+	zero := metav1.Duration{}
+	if obj.NodeStatusUpdateFrequency == zero {
+		obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 5 * time.Minute}
+	}
+
+	// These defaults override the recommended defaults from the apimachineryconfigv1alpha1 package that are applied automatically
+	// These client-connection defaults are specific to the cloud-controller-manager
+	if obj.Generic.ClientConnection.QPS == 0 {
+		obj.Generic.ClientConnection.QPS = 20
+	}
+	if obj.Generic.ClientConnection.Burst == 0 {
+		obj.Generic.ClientConnection.Burst = 30
+	}
+
+	// Use the default RecommendedDefaultGenericControllerManagerConfiguration options
+	kubectrlmgrconfigv1alpha1.RecommendedDefaultGenericControllerManagerConfiguration(&obj.Generic)
+	// Use the default RecommendedDefaultServiceControllerConfiguration options
+	serviceconfigv1alpha1.RecommendedDefaultServiceControllerConfiguration(&obj.ServiceController)
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/doc.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/doc.go
@ -0,0 +1,31 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Note: The referenced generic ComponentConfig packages with conversions
+// between the types (e.g. the external package) needs to be given as an
+// input to conversion-gen for it to find the native conversation funcs to
+// call.
+
+// +k8s:deepcopy-gen=package
+// +k8s:conversion-gen=k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config
+// +k8s:conversion-gen=k8s.io/component-base/config/v1alpha1
+// +k8s:conversion-gen=k8s.io/kubernetes/pkg/controller/apis/config/v1alpha1
+// +k8s:conversion-gen=k8s.io/kubernetes/pkg/controller/service/config/v1alpha1
+// +k8s:openapi-gen=true
+// +k8s:defaulter-gen=TypeMeta
+// +groupName=cloudcontrollermanager.config.k8s.io
+
+package v1alpha1 // import "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1"
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/register.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/register.go
@ -0,0 +1,52 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// GroupName is the group name use in this package
+const GroupName = "cloudcontrollermanager.config.k8s.io"
+
+var (
+	// GroupName is the group name use in this package
+	SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
+	// SchemeBuilder is the scheme builder with scheme init functions to run for this API package
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	// localSchemeBuilder ïs a pointer to SchemeBuilder instance. Using localSchemeBuilder
+	// defaulting and conversion init funcs are registered as well.
+	localSchemeBuilder = &SchemeBuilder
+	// AddToScheme is a global function that registers this API group & version to a scheme
+	AddToScheme = localSchemeBuilder.AddToScheme
+)
+
+func init() {
+	// We only register manually written functions here. The registration of the
+	// generated functions takes place in the generated files. The separation
+	// makes the code compile even when the generated files are missing.
+	localSchemeBuilder.Register(addDefaultingFuncs)
+}
+
+// addKnownTypes registers known types to the given scheme
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&CloudControllerManagerConfiguration{},
+	)
+	return nil
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/types.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/types.go
@ -0,0 +1,39 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubectrlmgrconfigv1alpha1 "k8s.io/kube-controller-manager/config/v1alpha1"
+)
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type CloudControllerManagerConfiguration struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// Generic holds configuration for a generic controller-manager
+	Generic kubectrlmgrconfigv1alpha1.GenericControllerManagerConfiguration
+	// KubeCloudSharedConfiguration holds configuration for shared related features
+	// both in cloud controller manager and kube-controller manager.
+	KubeCloudShared kubectrlmgrconfigv1alpha1.KubeCloudSharedConfiguration
+	// ServiceControllerConfiguration holds configuration for ServiceController
+	// related features.
+	ServiceController kubectrlmgrconfigv1alpha1.ServiceControllerConfiguration
+	// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
+	NodeStatusUpdateFrequency metav1.Duration
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/zz_generated.conversion.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/zz_generated.conversion.go
@ -0,0 +1,87 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by conversion-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	conversion "k8s.io/apimachinery/pkg/conversion"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	config "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config"
+	configv1alpha1 "k8s.io/kubernetes/pkg/controller/apis/config/v1alpha1"
+	serviceconfigv1alpha1 "k8s.io/kubernetes/pkg/controller/service/config/v1alpha1"
+)
+
+func init() {
+	localSchemeBuilder.Register(RegisterConversions)
+}
+
+// RegisterConversions adds conversion functions to the given scheme.
+// Public to allow building arbitrary schemes.
+func RegisterConversions(s *runtime.Scheme) error {
+	if err := s.AddGeneratedConversionFunc((*CloudControllerManagerConfiguration)(nil), (*config.CloudControllerManagerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_CloudControllerManagerConfiguration_To_config_CloudControllerManagerConfiguration(a.(*CloudControllerManagerConfiguration), b.(*config.CloudControllerManagerConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*config.CloudControllerManagerConfiguration)(nil), (*CloudControllerManagerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_config_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration(a.(*config.CloudControllerManagerConfiguration), b.(*CloudControllerManagerConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func autoConvert_v1alpha1_CloudControllerManagerConfiguration_To_config_CloudControllerManagerConfiguration(in *CloudControllerManagerConfiguration, out *config.CloudControllerManagerConfiguration, s conversion.Scope) error {
+	if err := configv1alpha1.Convert_v1alpha1_GenericControllerManagerConfiguration_To_config_GenericControllerManagerConfiguration(&in.Generic, &out.Generic, s); err != nil {
+		return err
+	}
+	if err := configv1alpha1.Convert_v1alpha1_KubeCloudSharedConfiguration_To_config_KubeCloudSharedConfiguration(&in.KubeCloudShared, &out.KubeCloudShared, s); err != nil {
+		return err
+	}
+	if err := serviceconfigv1alpha1.Convert_v1alpha1_ServiceControllerConfiguration_To_config_ServiceControllerConfiguration(&in.ServiceController, &out.ServiceController, s); err != nil {
+		return err
+	}
+	out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
+	return nil
+}
+
+// Convert_v1alpha1_CloudControllerManagerConfiguration_To_config_CloudControllerManagerConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_CloudControllerManagerConfiguration_To_config_CloudControllerManagerConfiguration(in *CloudControllerManagerConfiguration, out *config.CloudControllerManagerConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_CloudControllerManagerConfiguration_To_config_CloudControllerManagerConfiguration(in, out, s)
+}
+
+func autoConvert_config_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration(in *config.CloudControllerManagerConfiguration, out *CloudControllerManagerConfiguration, s conversion.Scope) error {
+	if err := configv1alpha1.Convert_config_GenericControllerManagerConfiguration_To_v1alpha1_GenericControllerManagerConfiguration(&in.Generic, &out.Generic, s); err != nil {
+		return err
+	}
+	if err := configv1alpha1.Convert_config_KubeCloudSharedConfiguration_To_v1alpha1_KubeCloudSharedConfiguration(&in.KubeCloudShared, &out.KubeCloudShared, s); err != nil {
+		return err
+	}
+	if err := serviceconfigv1alpha1.Convert_config_ServiceControllerConfiguration_To_v1alpha1_ServiceControllerConfiguration(&in.ServiceController, &out.ServiceController, s); err != nil {
+		return err
+	}
+	out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
+	return nil
+}
+
+// Convert_config_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration is an autogenerated conversion function.
+func Convert_config_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration(in *config.CloudControllerManagerConfiguration, out *CloudControllerManagerConfiguration, s conversion.Scope) error {
+	return autoConvert_config_CloudControllerManagerConfiguration_To_v1alpha1_CloudControllerManagerConfiguration(in, out, s)
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/zz_generated.deepcopy.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/zz_generated.deepcopy.go
@ -0,0 +1,54 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by deepcopy-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CloudControllerManagerConfiguration) DeepCopyInto(out *CloudControllerManagerConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.Generic.DeepCopyInto(&out.Generic)
+	in.KubeCloudShared.DeepCopyInto(&out.KubeCloudShared)
+	out.ServiceController = in.ServiceController
+	out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudControllerManagerConfiguration.
+func (in *CloudControllerManagerConfiguration) DeepCopy() *CloudControllerManagerConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(CloudControllerManagerConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CloudControllerManagerConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/zz_generated.defaults.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1/zz_generated.defaults.go
@ -0,0 +1,41 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by defaulter-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	configv1alpha1 "k8s.io/kubernetes/pkg/controller/apis/config/v1alpha1"
+)
+
+// RegisterDefaults adds defaulters functions to the given scheme.
+// Public to allow building arbitrary schemes.
+// All generated defaulters are covering - they call all nested defaulters.
+func RegisterDefaults(scheme *runtime.Scheme) error {
+	scheme.AddTypeDefaultingFunc(&CloudControllerManagerConfiguration{}, func(obj interface{}) {
+		SetObjectDefaults_CloudControllerManagerConfiguration(obj.(*CloudControllerManagerConfiguration))
+	})
+	return nil
+}
+
+func SetObjectDefaults_CloudControllerManagerConfiguration(in *CloudControllerManagerConfiguration) {
+	SetDefaults_CloudControllerManagerConfiguration(in)
+	configv1alpha1.SetDefaults_KubeCloudSharedConfiguration(&in.KubeCloudShared)
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/zz_generated.deepcopy.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/zz_generated.deepcopy.go
@ -0,0 +1,54 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by deepcopy-gen. DO NOT EDIT.
+
+package config
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CloudControllerManagerConfiguration) DeepCopyInto(out *CloudControllerManagerConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.Generic.DeepCopyInto(&out.Generic)
+	out.KubeCloudShared = in.KubeCloudShared
+	out.ServiceController = in.ServiceController
+	out.NodeStatusUpdateFrequency = in.NodeStatusUpdateFrequency
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudControllerManagerConfiguration.
+func (in *CloudControllerManagerConfiguration) DeepCopy() *CloudControllerManagerConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(CloudControllerManagerConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CloudControllerManagerConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/config/BUILD
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/config/BUILD
@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/cloud-controller-manager/app/apis/config:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/config/config.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/config/config.go
@ -0,0 +1,81 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	apiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/record"
+	ccmconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+// Config is the main context object for the cloud controller manager.
+type Config struct {
+	ComponentConfig ccmconfig.CloudControllerManagerConfiguration
+
+	SecureServing *apiserver.SecureServingInfo
+	// LoopbackClientConfig is a config for a privileged loopback connection
+	LoopbackClientConfig *restclient.Config
+
+	// TODO: remove deprecated insecure serving
+	InsecureServing *apiserver.DeprecatedInsecureServingInfo
+	Authentication  apiserver.AuthenticationInfo
+	Authorization   apiserver.AuthorizationInfo
+
+	// the general kube client
+	Client *clientset.Clientset
+
+	// the client only used for leader election
+	LeaderElectionClient *clientset.Clientset
+
+	// the rest config for the master
+	Kubeconfig *restclient.Config
+
+	// the event sink
+	EventRecorder record.EventRecorder
+
+	// ClientBuilder will provide a client for this controller to use
+	ClientBuilder controller.ControllerClientBuilder
+
+	// VersionedClient will provide a client for informers
+	VersionedClient clientset.Interface
+
+	// SharedInformers gives access to informers for the controller.
+	SharedInformers informers.SharedInformerFactory
+}
+
+type completedConfig struct {
+	*Config
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() *CompletedConfig {
+	cc := completedConfig{c}
+
+	apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization)
+
+	return &CompletedConfig{&cc}
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/controllermanager.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/controllermanager.go
@ -0,0 +1,283 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/util/term"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	cloudprovider "k8s.io/cloud-provider"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/cli/globalflag"
+	"k8s.io/component-base/version"
+	"k8s.io/klog"
+	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
+	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
+	"k8s.io/kubernetes/pkg/util/configz"
+	utilflag "k8s.io/kubernetes/pkg/util/flag"
+	"k8s.io/kubernetes/pkg/version/verflag"
+)
+
+const (
+	// ControllerStartJitter is the jitter value used when starting controller managers.
+	ControllerStartJitter = 1.0
+	// ConfigzName is the name used for register cloud-controller manager /configz, same with GroupName.
+	ConfigzName = "cloudcontrollermanager.config.k8s.io"
+)
+
+// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
+func NewCloudControllerManagerCommand() *cobra.Command {
+	s, err := options.NewCloudControllerManagerOptions()
+	if err != nil {
+		klog.Fatalf("unable to initialize command options: %v", err)
+	}
+
+	cmd := &cobra.Command{
+		Use: "cloud-controller-manager",
+		Long: `The Cloud controller manager is a daemon that embeds
+the cloud specific control loops shipped with Kubernetes.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			verflag.PrintAndExitIfRequested()
+			utilflag.PrintFlags(cmd.Flags())
+
+			c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
+
+			if err := Run(c.Complete(), wait.NeverStop); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
+
+		},
+	}
+
+	fs := cmd.Flags()
+	namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
+	verflag.AddFlags(namedFlagSets.FlagSet("global"))
+	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
+
+	if flag.CommandLine.Lookup("cloud-provider-gce-lb-src-cidrs") != nil {
+		// hoist this flag from the global flagset to preserve the commandline until
+		// the gce cloudprovider is removed.
+		globalflag.Register(namedFlagSets.FlagSet("generic"), "cloud-provider-gce-lb-src-cidrs")
+	}
+	for _, f := range namedFlagSets.FlagSets {
+		fs.AddFlagSet(f)
+	}
+	usageFmt := "Usage:\n  %s\n"
+	cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
+	cmd.SetUsageFunc(func(cmd *cobra.Command) error {
+		fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine())
+		cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols)
+		return nil
+	})
+	cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
+		fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
+		cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
+	})
+
+	return cmd
+}
+
+// Run runs the ExternalCMServer.  This should never exit.
+func Run(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}) error {
+	// To help debugging, immediately log version
+	klog.Infof("Version: %+v", version.Get())
+
+	cloud, err := cloudprovider.InitCloudProvider(c.ComponentConfig.KubeCloudShared.CloudProvider.Name, c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
+	if err != nil {
+		klog.Fatalf("Cloud provider could not be initialized: %v", err)
+	}
+	if cloud == nil {
+		klog.Fatalf("cloud provider is nil")
+	}
+
+	if !cloud.HasClusterID() {
+		if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
+			klog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
+		} else {
+			klog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
+		}
+	}
+
+	// setup /configz endpoint
+	if cz, err := configz.New(ConfigzName); err == nil {
+		cz.Set(c.ComponentConfig)
+	} else {
+		klog.Errorf("unable to register configz: %v", err)
+	}
+
+	// Setup any health checks we will want to use.
+	var checks []healthz.HealthChecker
+	var electionChecker *leaderelection.HealthzAdaptor
+	if c.ComponentConfig.Generic.LeaderElection.LeaderElect {
+		electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20)
+		checks = append(checks, electionChecker)
+	}
+
+	// Start the controller manager HTTP server
+	if c.SecureServing != nil {
+		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
+		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication)
+		// TODO: handle stoppedCh returned by c.SecureServing.Serve
+		if _, err := c.SecureServing.Serve(handler, 0, stopCh); err != nil {
+			return err
+		}
+	}
+	if c.InsecureServing != nil {
+		unsecuredMux := genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...)
+		insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}}
+		handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn)
+		if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil {
+			return err
+		}
+	}
+
+	run := func(ctx context.Context) {
+		if err := startControllers(c, ctx.Done(), cloud, newControllerInitializers()); err != nil {
+			klog.Fatalf("error running controllers: %v", err)
+		}
+	}
+
+	if !c.ComponentConfig.Generic.LeaderElection.LeaderElect {
+		run(context.TODO())
+		panic("unreachable")
+	}
+
+	// Identity used to distinguish between multiple cloud controller manager instances
+	id, err := os.Hostname()
+	if err != nil {
+		return err
+	}
+	// add a uniquifier so that two processes on the same host don't accidentally both become active
+	id = id + "_" + string(uuid.NewUUID())
+
+	// Lock required for leader election
+	rl, err := resourcelock.New(c.ComponentConfig.Generic.LeaderElection.ResourceLock,
+		c.ComponentConfig.Generic.LeaderElection.ResourceNamespace,
+		c.ComponentConfig.Generic.LeaderElection.ResourceName,
+		c.LeaderElectionClient.CoreV1(),
+		c.LeaderElectionClient.CoordinationV1(),
+		resourcelock.ResourceLockConfig{
+			Identity:      id,
+			EventRecorder: c.EventRecorder,
+		})
+	if err != nil {
+		klog.Fatalf("error creating lock: %v", err)
+	}
+
+	// Try and become the leader and start cloud controller manager loops
+	leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
+		Lock:          rl,
+		LeaseDuration: c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration,
+		RenewDeadline: c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration,
+		RetryPeriod:   c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: run,
+			OnStoppedLeading: func() {
+				klog.Fatalf("leaderelection lost")
+			},
+		},
+		WatchDog: electionChecker,
+		Name:     "cloud-controller-manager",
+	})
+	panic("unreachable")
+}
+
+// startControllers starts the cloud specific controller loops.
+func startControllers(c *cloudcontrollerconfig.CompletedConfig, stopCh <-chan struct{}, cloud cloudprovider.Interface, controllers map[string]initFunc) error {
+	// Initialize the cloud provider with a reference to the clientBuilder
+	cloud.Initialize(c.ClientBuilder, stopCh)
+	// Set the informer on the user cloud object
+	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
+		informerUserCloud.SetInformers(c.SharedInformers)
+	}
+
+	for controllerName, initFn := range controllers {
+		if !genericcontrollermanager.IsControllerEnabled(controllerName, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) {
+			klog.Warningf("%q is disabled", controllerName)
+			continue
+		}
+
+		klog.V(1).Infof("Starting %q", controllerName)
+		_, started, err := initFn(c, cloud, stopCh)
+		if err != nil {
+			klog.Errorf("Error starting %q", controllerName)
+			return err
+		}
+		if !started {
+			klog.Warningf("Skipping %q", controllerName)
+			continue
+		}
+		klog.Infof("Started %q", controllerName)
+
+		time.Sleep(wait.Jitter(c.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter))
+	}
+
+	// If apiserver is not running we should wait for some time and fail only then. This is particularly
+	// important when we start apiserver and controller manager at the same time.
+	if err := genericcontrollermanager.WaitForAPIServer(c.VersionedClient, 10*time.Second); err != nil {
+		klog.Fatalf("Failed to wait for apiserver being healthy: %v", err)
+	}
+
+	c.SharedInformers.Start(stopCh)
+
+	select {}
+}
+
+// initFunc is used to launch a particular controller.  It may run additional "should I activate checks".
+// Any error returned will cause the controller process to `Fatal`
+// The bool indicates whether the controller was enabled.
+type initFunc func(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stop <-chan struct{}) (debuggingHandler http.Handler, enabled bool, err error)
+
+// KnownControllers indicate the default controller we are known.
+func KnownControllers() []string {
+	ret := sets.StringKeySet(newControllerInitializers())
+	return ret.List()
+}
+
+// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
+var ControllersDisabledByDefault = sets.NewString()
+
+// newControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
+// paired to their initFunc.  This allows for structured downstream composition and subdivision.
+func newControllerInitializers() map[string]initFunc {
+	controllers := map[string]initFunc{}
+	controllers["cloud-node"] = startCloudNodeController
+	controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController
+	controllers["service"] = startServiceController
+	controllers["route"] = startRouteController
+	return controllers
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/core.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/core.go
@ -0,0 +1,157 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package app implements a server that runs a set of active
+// components.  This includes node controllers, service and
+// route controller, and so on.
+//
+package app
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+
+	cloudprovider "k8s.io/cloud-provider"
+	"k8s.io/klog"
+	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
+	cloudcontrollers "k8s.io/kubernetes/pkg/controller/cloud"
+	routecontroller "k8s.io/kubernetes/pkg/controller/route"
+	servicecontroller "k8s.io/kubernetes/pkg/controller/service"
+	netutils "k8s.io/utils/net"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	kubefeatures "k8s.io/kubernetes/pkg/features"
+)
+
+func startCloudNodeController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
+	// Start the CloudNodeController
+	nodeController := cloudcontrollers.NewCloudNodeController(
+		ctx.SharedInformers.Core().V1().Nodes(),
+		// cloud node controller uses existing cluster role from node-controller
+		ctx.ClientBuilder.ClientOrDie("node-controller"),
+		cloud,
+		ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration)
+
+	go nodeController.Run(stopCh)
+
+	return nil, true, nil
+}
+
+func startCloudNodeLifecycleController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
+	// Start the cloudNodeLifecycleController
+	cloudNodeLifecycleController, err := cloudcontrollers.NewCloudNodeLifecycleController(
+		ctx.SharedInformers.Core().V1().Nodes(),
+		// cloud node lifecycle controller uses existing cluster role from node-controller
+		ctx.ClientBuilder.ClientOrDie("node-controller"),
+		cloud,
+		ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
+	)
+	if err != nil {
+		klog.Warningf("failed to start cloud node lifecycle controller: %s", err)
+		return nil, false, nil
+	}
+
+	go cloudNodeLifecycleController.Run(stopCh)
+
+	return nil, true, nil
+}
+
+func startServiceController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
+	// Start the service controller
+	serviceController, err := servicecontroller.New(
+		cloud,
+		ctx.ClientBuilder.ClientOrDie("service-controller"),
+		ctx.SharedInformers.Core().V1().Services(),
+		ctx.SharedInformers.Core().V1().Nodes(),
+		ctx.ComponentConfig.KubeCloudShared.ClusterName,
+	)
+	if err != nil {
+		// This error shouldn't fail. It lives like this as a legacy.
+		klog.Errorf("Failed to start service controller: %v", err)
+		return nil, false, nil
+	}
+
+	go serviceController.Run(stopCh, int(ctx.ComponentConfig.ServiceController.ConcurrentServiceSyncs))
+
+	return nil, true, nil
+}
+
+func startRouteController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) {
+	if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs || !ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes {
+		klog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes)
+		return nil, false, nil
+	}
+
+	// If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller
+	routes, ok := cloud.Routes()
+	if !ok {
+		klog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.")
+		return nil, false, nil
+	}
+
+	// failure: bad cidrs in config
+	clusterCIDRs, dualStack, err := processCIDRs(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// failure: more than one cidr and dual stack is not enabled
+	if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.IPv6DualStack) {
+		return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack feature is not enabled", len(clusterCIDRs))
+	}
+
+	// failure: more than one cidr but they are not configured as dual stack
+	if len(clusterCIDRs) > 1 && !dualStack {
+		return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs))
+	}
+
+	// failure: more than cidrs is not allowed even with dual stack
+	if len(clusterCIDRs) > 2 {
+		return nil, false, fmt.Errorf("length of clusterCIDRs is:%v more than max allowed of 2", len(clusterCIDRs))
+	}
+
+	routeController := routecontroller.New(
+		routes,
+		ctx.ClientBuilder.ClientOrDie("route-controller"),
+		ctx.SharedInformers.Core().V1().Nodes(),
+		ctx.ComponentConfig.KubeCloudShared.ClusterName,
+		clusterCIDRs,
+	)
+	go routeController.Run(stopCh, ctx.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration)
+
+	return nil, true, nil
+}
+
+// processCIDRs is a helper function that works on a comma separated cidrs and returns
+// a list of typed cidrs
+// a flag if cidrs represents a dual stack
+// error if failed to parse any of the cidrs
+func processCIDRs(cidrsList string) ([]*net.IPNet, bool, error) {
+	cidrsSplit := strings.Split(strings.TrimSpace(cidrsList), ",")
+
+	cidrs, err := netutils.ParseCIDRs(cidrsSplit)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// if cidrs has an error then the previous call will fail
+	// safe to ignore error checking on next call
+	dualstack, _ := netutils.IsDualStackCIDRs(cidrs)
+
+	return cidrs, dualstack, nil
+}
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/options/BUILD
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/options/BUILD
@ -0,0 +1,66 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["options.go"],
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
+    deps = [
+        "//cmd/cloud-controller-manager/app/apis/config:go_default_library",
+        "//cmd/cloud-controller-manager/app/apis/config/scheme:go_default_library",
+        "//cmd/cloud-controller-manager/app/apis/config/v1alpha1:go_default_library",
+        "//cmd/cloud-controller-manager/app/config:go_default_library",
+        "//cmd/controller-manager/app/options:go_default_library",
+        "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/master/ports:go_default_library",
+        "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//staging/src/k8s.io/client-go/informers:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//staging/src/k8s.io/client-go/rest:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/record:go_default_library",
+        "//staging/src/k8s.io/component-base/cli/flag:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["options_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
+        "//pkg/controller/apis/config:go_default_library",
+        "//pkg/controller/service/config:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",
+        "//staging/src/k8s.io/component-base/config:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+    ],
+)
--- a/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/options/options.go
+++ b/vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/options/options.go
@ -0,0 +1,273 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/record"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
+	ccmconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config"
+	ccmconfigscheme "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme"
+	ccmconfigv1alpha1 "k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1"
+	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/master/ports"
+
+	// add the kubernetes feature gates
+	_ "k8s.io/kubernetes/pkg/features"
+)
+
+const (
+	// CloudControllerManagerUserAgent is the userAgent name when starting cloud-controller managers.
+	CloudControllerManagerUserAgent = "cloud-controller-manager"
+	// DefaultInsecureCloudControllerManagerPort is the default insecure cloud-controller manager port.
+	DefaultInsecureCloudControllerManagerPort = 0
+)
+
+// CloudControllerManagerOptions is the main context object for the controller manager.
+type CloudControllerManagerOptions struct {
+	Generic           *cmoptions.GenericControllerManagerConfigurationOptions
+	KubeCloudShared   *cmoptions.KubeCloudSharedOptions
+	ServiceController *cmoptions.ServiceControllerOptions
+
+	SecureServing *apiserveroptions.SecureServingOptionsWithLoopback
+	// TODO: remove insecure serving mode
+	InsecureServing *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback
+	Authentication  *apiserveroptions.DelegatingAuthenticationOptions
+	Authorization   *apiserveroptions.DelegatingAuthorizationOptions
+
+	Master     string
+	Kubeconfig string
+
+	// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
+	NodeStatusUpdateFrequency metav1.Duration
+}
+
+// NewCloudControllerManagerOptions creates a new ExternalCMServer with a default config.
+func NewCloudControllerManagerOptions() (*CloudControllerManagerOptions, error) {
+	componentConfig, err := NewDefaultComponentConfig(DefaultInsecureCloudControllerManagerPort)
+	if err != nil {
+		return nil, err
+	}
+
+	s := CloudControllerManagerOptions{
+		Generic:         cmoptions.NewGenericControllerManagerConfigurationOptions(&componentConfig.Generic),
+		KubeCloudShared: cmoptions.NewKubeCloudSharedOptions(&componentConfig.KubeCloudShared),
+		ServiceController: &cmoptions.ServiceControllerOptions{
+			ServiceControllerConfiguration: &componentConfig.ServiceController,
+		},
+		SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(),
+		InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{
+			BindAddress: net.ParseIP(componentConfig.Generic.Address),
+			BindPort:    int(componentConfig.Generic.Port),
+			BindNetwork: "tcp",
+		}).WithLoopback(),
+		Authentication:            apiserveroptions.NewDelegatingAuthenticationOptions(),
+		Authorization:             apiserveroptions.NewDelegatingAuthorizationOptions(),
+		NodeStatusUpdateFrequency: componentConfig.NodeStatusUpdateFrequency,
+	}
+
+	s.Authentication.RemoteKubeConfigFileOptional = true
+	s.Authorization.RemoteKubeConfigFileOptional = true
+	s.Authorization.AlwaysAllowPaths = []string{"/healthz"}
+
+	// Set the PairName but leave certificate directory blank to generate in-memory by default
+	s.SecureServing.ServerCert.CertDirectory = ""
+	s.SecureServing.ServerCert.PairName = "cloud-controller-manager"
+	s.SecureServing.BindPort = ports.CloudControllerManagerPort
+
+	s.Generic.LeaderElection.ResourceName = "cloud-controller-manager"
+	s.Generic.LeaderElection.ResourceNamespace = "kube-system"
+
+	return &s, nil
+}
+
+// NewDefaultComponentConfig returns cloud-controller manager configuration object.
+func NewDefaultComponentConfig(insecurePort int32) (*ccmconfig.CloudControllerManagerConfiguration, error) {
+	versioned := &ccmconfigv1alpha1.CloudControllerManagerConfiguration{}
+	ccmconfigscheme.Scheme.Default(versioned)
+
+	internal := &ccmconfig.CloudControllerManagerConfiguration{}
+	if err := ccmconfigscheme.Scheme.Convert(versioned, internal, nil); err != nil {
+		return nil, err
+	}
+	internal.Generic.Port = insecurePort
+	return internal, nil
+}
+
+// Flags returns flags for a specific APIServer by section name
+func (o *CloudControllerManagerOptions) Flags(allControllers, disabledByDefaultControllers []string) cliflag.NamedFlagSets {
+	fss := cliflag.NamedFlagSets{}
+	o.Generic.AddFlags(&fss, allControllers, disabledByDefaultControllers)
+	o.KubeCloudShared.AddFlags(fss.FlagSet("generic"))
+	o.ServiceController.AddFlags(fss.FlagSet("service controller"))
+
+	o.SecureServing.AddFlags(fss.FlagSet("secure serving"))
+	o.InsecureServing.AddUnqualifiedFlags(fss.FlagSet("insecure serving"))
+	o.Authentication.AddFlags(fss.FlagSet("authentication"))
+	o.Authorization.AddFlags(fss.FlagSet("authorization"))
+
+	fs := fss.FlagSet("misc")
+	fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
+	fs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
+	fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
+
+	utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))
+
+	return fss
+}
+
+// ApplyTo fills up cloud controller manager config with options.
+func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config, userAgent string) error {
+	var err error
+	if err = o.Generic.ApplyTo(&c.ComponentConfig.Generic); err != nil {
+		return err
+	}
+	if err = o.KubeCloudShared.ApplyTo(&c.ComponentConfig.KubeCloudShared); err != nil {
+		return err
+	}
+	if err = o.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil {
+		return err
+	}
+	if err = o.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil {
+		return err
+	}
+	if err = o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil {
+		return err
+	}
+	if o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil {
+		if err = o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil {
+			return err
+		}
+		if err = o.Authorization.ApplyTo(&c.Authorization); err != nil {
+			return err
+		}
+	}
+
+	c.Kubeconfig, err = clientcmd.BuildConfigFromFlags(o.Master, o.Kubeconfig)
+	if err != nil {
+		return err
+	}
+	c.Kubeconfig.DisableCompression = true
+	c.Kubeconfig.ContentConfig.ContentType = o.Generic.ClientConnection.ContentType
+	c.Kubeconfig.QPS = o.Generic.ClientConnection.QPS
+	c.Kubeconfig.Burst = int(o.Generic.ClientConnection.Burst)
+
+	c.Client, err = clientset.NewForConfig(restclient.AddUserAgent(c.Kubeconfig, userAgent))
+	if err != nil {
+		return err
+	}
+
+	c.LeaderElectionClient = clientset.NewForConfigOrDie(restclient.AddUserAgent(c.Kubeconfig, "leader-election"))
+
+	c.EventRecorder = createRecorder(c.Client, userAgent)
+
+	rootClientBuilder := controller.SimpleControllerClientBuilder{
+		ClientConfig: c.Kubeconfig,
+	}
+	if c.ComponentConfig.KubeCloudShared.UseServiceAccountCredentials {
+		c.ClientBuilder = controller.SAControllerClientBuilder{
+			ClientConfig:         restclient.AnonymousClientConfig(c.Kubeconfig),
+			CoreClient:           c.Client.CoreV1(),
+			AuthenticationClient: c.Client.AuthenticationV1(),
+			Namespace:            metav1.NamespaceSystem,
+		}
+	} else {
+		c.ClientBuilder = rootClientBuilder
+	}
+	c.VersionedClient = rootClientBuilder.ClientOrDie("shared-informers")
+	c.SharedInformers = informers.NewSharedInformerFactory(c.VersionedClient, resyncPeriod(c)())
+
+	// sync back to component config
+	// TODO: find more elegant way than syncing back the values.
+	c.ComponentConfig.Generic.Port = int32(o.InsecureServing.BindPort)
+	c.ComponentConfig.Generic.Address = o.InsecureServing.BindAddress.String()
+
+	c.ComponentConfig.NodeStatusUpdateFrequency = o.NodeStatusUpdateFrequency
+
+	return nil
+}
+
+// Validate is used to validate config before launching the cloud controller manager
+func (o *CloudControllerManagerOptions) Validate(allControllers, disabledByDefaultControllers []string) error {
+	errors := []error{}
+
+	errors = append(errors, o.Generic.Validate(allControllers, disabledByDefaultControllers)...)
+	errors = append(errors, o.KubeCloudShared.Validate()...)
+	errors = append(errors, o.ServiceController.Validate()...)
+	errors = append(errors, o.SecureServing.Validate()...)
+	errors = append(errors, o.InsecureServing.Validate()...)
+	errors = append(errors, o.Authentication.Validate()...)
+	errors = append(errors, o.Authorization.Validate()...)
+
+	if len(o.KubeCloudShared.CloudProvider.Name) == 0 {
+		errors = append(errors, fmt.Errorf("--cloud-provider cannot be empty"))
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// resyncPeriod computes the time interval a shared informer waits before resyncing with the api server
+func resyncPeriod(c *cloudcontrollerconfig.Config) func() time.Duration {
+	return func() time.Duration {
+		factor := rand.Float64() + 1
+		return time.Duration(float64(c.ComponentConfig.Generic.MinResyncPeriod.Nanoseconds()) * factor)
+	}
+}
+
+// Config return a cloud controller manager config objective
+func (o *CloudControllerManagerOptions) Config(allControllers, disabledByDefaultControllers []string) (*cloudcontrollerconfig.Config, error) {
+	if err := o.Validate(allControllers, disabledByDefaultControllers); err != nil {
+		return nil, err
+	}
+
+	if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
+		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+
+	c := &cloudcontrollerconfig.Config{}
+	if err := o.ApplyTo(c, CloudControllerManagerUserAgent); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func createRecorder(kubeClient clientset.Interface, userAgent string) record.EventRecorder {
+	eventBroadcaster := record.NewBroadcaster()
+	eventBroadcaster.StartLogging(klog.Infof)
+	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
+	// TODO: remove dependence on the legacyscheme
+	return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: userAgent})
+}
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@ -757,6 +757,7 @@ github.com/rancher/kine/pkg/broadcaster
 # github.com/rancher/remotedialer v0.2.0
 github.com/rancher/remotedialer
 # github.com/rancher/wrangler v0.2.0
+github.com/rancher/wrangler/pkg/start
 github.com/rancher/wrangler/pkg/signals
 github.com/rancher/wrangler/pkg/controller-gen
 github.com/rancher/wrangler/pkg/controller-gen/args
@ -770,7 +771,6 @@ github.com/rancher/wrangler/pkg/generic
 github.com/rancher/wrangler/pkg/schemes
 github.com/rancher/wrangler/pkg/crd
 github.com/rancher/wrangler/pkg/leader
-github.com/rancher/wrangler/pkg/start
 github.com/rancher/wrangler/pkg/condition
 github.com/rancher/wrangler/pkg/relatedresource
 github.com/rancher/wrangler/pkg/slice
@ -779,14 +779,14 @@ github.com/rancher/wrangler/pkg/apply/injectors
 github.com/rancher/wrangler/pkg/gvk
 github.com/rancher/wrangler/pkg/name
 # github.com/rancher/wrangler-api v0.2.0
+github.com/rancher/wrangler-api/pkg/generated/controllers/core
 github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1
+github.com/rancher/wrangler-api/pkg/generated/controllers/rbac
 github.com/rancher/wrangler-api/pkg/generated/controllers/apps
 github.com/rancher/wrangler-api/pkg/generated/controllers/batch
-github.com/rancher/wrangler-api/pkg/generated/controllers/core
-github.com/rancher/wrangler-api/pkg/generated/controllers/rbac
 github.com/rancher/wrangler-api/pkg/generated/controllers/apps/v1
-github.com/rancher/wrangler-api/pkg/generated/controllers/batch/v1
 github.com/rancher/wrangler-api/pkg/generated/controllers/rbac/v1
+github.com/rancher/wrangler-api/pkg/generated/controllers/batch/v1
 # github.com/robfig/cron v1.1.0
 github.com/robfig/cron
 # github.com/rootless-containers/rootlesskit v0.6.0
@ -1184,23 +1184,23 @@ k8s.io/apimachinery/pkg/fields
 k8s.io/apimachinery/pkg/watch
 k8s.io/apimachinery/pkg/runtime
 k8s.io/apimachinery/pkg/runtime/schema
+k8s.io/apimachinery/pkg/types
 k8s.io/apimachinery/pkg/api/errors
 k8s.io/apimachinery/pkg/apis/meta/v1/unstructured
 k8s.io/apimachinery/pkg/util/yaml
 k8s.io/apimachinery/pkg/runtime/serializer
 k8s.io/apimachinery/pkg/util/runtime
-k8s.io/apimachinery/pkg/types
 k8s.io/apimachinery/pkg/labels
 k8s.io/apimachinery/pkg/api/equality
 k8s.io/apimachinery/pkg/util/intstr
+k8s.io/apimachinery/pkg/util/errors
+k8s.io/apimachinery/pkg/util/validation
 k8s.io/apimachinery/pkg/util/sets
 k8s.io/apimachinery/pkg/util/wait
 k8s.io/apimachinery/pkg/util/strategicpatch
 k8s.io/apimachinery/pkg/api/resource
 k8s.io/apimachinery/pkg/conversion
 k8s.io/apimachinery/pkg/selection
-k8s.io/apimachinery/pkg/util/errors
-k8s.io/apimachinery/pkg/util/validation
 k8s.io/apimachinery/pkg/conversion/queryparams
 k8s.io/apimachinery/pkg/util/naming
 k8s.io/apimachinery/pkg/api/validation/path
@ -1255,13 +1255,13 @@ k8s.io/apiserver/pkg/authorization/authorizer
 k8s.io/apiserver/pkg/authorization/authorizerfactory
 k8s.io/apiserver/pkg/authentication/user
 k8s.io/apiserver/pkg/apis/audit
+k8s.io/apiserver/pkg/util/term
 k8s.io/apiserver/pkg/admission
 k8s.io/apiserver/pkg/endpoints/openapi
 k8s.io/apiserver/pkg/features
 k8s.io/apiserver/pkg/server/filters
 k8s.io/apiserver/pkg/server/options
 k8s.io/apiserver/pkg/storage/etcd3/preflight
-k8s.io/apiserver/pkg/util/term
 k8s.io/apiserver/pkg/util/webhook
 k8s.io/apiserver/pkg/endpoints/filters
 k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle
@ -1365,9 +1365,9 @@ k8s.io/cli-runtime/pkg/kustomize/k8sdeps/transformer/hash
 k8s.io/cli-runtime/pkg/kustomize/k8sdeps/transformer/patch
 k8s.io/cli-runtime/pkg/kustomize/k8sdeps/kv
 # k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.16.0-k3s.1
+k8s.io/client-go/tools/clientcmd
 k8s.io/client-go/util/cert
 k8s.io/client-go/kubernetes
-k8s.io/client-go/tools/clientcmd
 k8s.io/client-go/transport
 k8s.io/client-go/tools/clientcmd/api
 k8s.io/client-go/discovery
@ -1381,8 +1381,15 @@ k8s.io/client-go/kubernetes/typed/core/v1
 k8s.io/client-go/tools/portforward
 k8s.io/client-go/tools/remotecommand
 k8s.io/client-go/transport/spdy
-k8s.io/client-go/util/keyutil
+k8s.io/client-go/informers
+k8s.io/client-go/informers/core
+k8s.io/client-go/kubernetes/scheme
+k8s.io/client-go/informers/core/v1
 k8s.io/client-go/listers/core/v1
+k8s.io/client-go/tools/auth
+k8s.io/client-go/tools/clientcmd/api/latest
+k8s.io/client-go/util/homedir
+k8s.io/client-go/util/keyutil
 k8s.io/client-go/kubernetes/typed/admissionregistration/v1
 k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1
 k8s.io/client-go/kubernetes/typed/apps/v1beta1
@ -1419,14 +1426,13 @@ k8s.io/client-go/kubernetes/typed/settings/v1alpha1
 k8s.io/client-go/kubernetes/typed/storage/v1
 k8s.io/client-go/kubernetes/typed/storage/v1alpha1
 k8s.io/client-go/kubernetes/typed/storage/v1beta1
-k8s.io/client-go/tools/auth
-k8s.io/client-go/tools/clientcmd/api/latest
-k8s.io/client-go/util/homedir
-k8s.io/client-go/informers
 k8s.io/client-go/tools/record
 k8s.io/client-go/util/certificate
 k8s.io/client-go/util/connrotation
 k8s.io/client-go/tools/metrics
+k8s.io/client-go/informers/rbac
+k8s.io/client-go/tools/leaderelection
+k8s.io/client-go/tools/leaderelection/resourcelock
 k8s.io/client-go/discovery/cached
 k8s.io/client-go/discovery/cached/memory
 k8s.io/client-go/dynamic
@ -1434,25 +1440,18 @@ k8s.io/client-go/metadata
 k8s.io/client-go/metadata/metadatainformer
 k8s.io/client-go/restmapper
 k8s.io/client-go/scale
-k8s.io/client-go/tools/leaderelection
-k8s.io/client-go/tools/leaderelection/resourcelock
-k8s.io/client-go/kubernetes/scheme
 k8s.io/client-go/pkg/version
 k8s.io/client-go/plugin/pkg/client/auth/exec
 k8s.io/client-go/rest/watch
 k8s.io/client-go/util/workqueue
 k8s.io/client-go/tools/pager
 k8s.io/client-go/util/retry
-k8s.io/client-go/informers/core/v1
 k8s.io/client-go/informers/apps
 k8s.io/client-go/informers/batch
-k8s.io/client-go/informers/core
-k8s.io/client-go/informers/rbac
 k8s.io/client-go/informers/apps/v1
 k8s.io/client-go/listers/apps/v1
 k8s.io/client-go/tools/reference
 k8s.io/client-go/util/exec
-k8s.io/client-go/tools/clientcmd/api/v1
 k8s.io/client-go/informers/admissionregistration
 k8s.io/client-go/informers/auditregistration
 k8s.io/client-go/informers/autoscaling
@ -1468,10 +1467,15 @@ k8s.io/client-go/informers/policy
 k8s.io/client-go/informers/scheduling
 k8s.io/client-go/informers/settings
 k8s.io/client-go/informers/storage
+k8s.io/client-go/tools/clientcmd/api/v1
 k8s.io/client-go/tools/record/util
 k8s.io/client-go/informers/discovery/v1alpha1
 k8s.io/client-go/util/certificate/csr
 k8s.io/client-go/listers/storage/v1beta1
+k8s.io/client-go/informers/rbac/v1
+k8s.io/client-go/listers/rbac/v1
+k8s.io/client-go/informers/rbac/v1alpha1
+k8s.io/client-go/informers/rbac/v1beta1
 k8s.io/client-go/scale/scheme/autoscalingv1
 k8s.io/client-go/metadata/metadatalister
 k8s.io/client-go/scale/scheme
@ -1483,8 +1487,6 @@ k8s.io/client-go/scale/scheme/extensionsv1beta1
 k8s.io/client-go/tools/watch
 k8s.io/client-go/informers/certificates/v1beta1
 k8s.io/client-go/listers/certificates/v1beta1
-k8s.io/client-go/informers/rbac/v1
-k8s.io/client-go/listers/rbac/v1
 k8s.io/client-go/informers/policy/v1beta1
 k8s.io/client-go/listers/policy/v1beta1
 k8s.io/client-go/listers/discovery/v1alpha1
@ -1507,8 +1509,6 @@ k8s.io/client-go/informers/apps/v1beta1
 k8s.io/client-go/informers/apps/v1beta2
 k8s.io/client-go/informers/batch/v1beta1
 k8s.io/client-go/informers/batch/v2alpha1
-k8s.io/client-go/informers/rbac/v1alpha1
-k8s.io/client-go/informers/rbac/v1beta1
 k8s.io/client-go/informers/admissionregistration/v1
 k8s.io/client-go/informers/admissionregistration/v1beta1
 k8s.io/client-go/informers/auditregistration/v1alpha1
@ -1529,13 +1529,13 @@ k8s.io/client-go/informers/storage/v1alpha1
 k8s.io/client-go/listers/settings/v1alpha1
 k8s.io/client-go/listers/scheduling/v1
 k8s.io/client-go/listers/node/v1beta1
+k8s.io/client-go/listers/rbac/v1alpha1
+k8s.io/client-go/listers/rbac/v1beta1
 k8s.io/client-go/third_party/forked/golang/template
 k8s.io/client-go/listers/apps/v1beta1
 k8s.io/client-go/listers/apps/v1beta2
 k8s.io/client-go/listers/batch/v1beta1
 k8s.io/client-go/listers/batch/v2alpha1
-k8s.io/client-go/listers/rbac/v1alpha1
-k8s.io/client-go/listers/rbac/v1beta1
 k8s.io/client-go/listers/admissionregistration/v1beta1
 k8s.io/client-go/listers/admissionregistration/v1
 k8s.io/client-go/listers/auditregistration/v1alpha1
@ -1585,8 +1585,8 @@ k8s.io/component-base/metrics/prometheus/restclient
 k8s.io/component-base/config
 k8s.io/component-base/metrics/legacyregistry
 k8s.io/component-base/cli/globalflag
-k8s.io/component-base/metrics/prometheus/workqueue
 k8s.io/component-base/version
+k8s.io/component-base/metrics/prometheus/workqueue
 k8s.io/component-base/metrics
 k8s.io/component-base/featuregate
 k8s.io/component-base/config/v1alpha1
@ -1741,6 +1741,7 @@ k8s.io/kubernetes/cmd/kubelet/app
 k8s.io/kubernetes/pkg/client/metrics/prometheus
 k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes
 k8s.io/kubernetes/pkg/version/prometheus
+k8s.io/kubernetes/cmd/cloud-controller-manager/app
 k8s.io/kubernetes/cmd/kube-apiserver/app
 k8s.io/kubernetes/cmd/kube-controller-manager/app
 k8s.io/kubernetes/cmd/kube-scheduler/app
@ -1880,6 +1881,12 @@ k8s.io/kubernetes/pkg/volume/secret
 k8s.io/kubernetes/pkg/volume/util/hostutil
 k8s.io/kubernetes/pkg/volume/util/subpath
 k8s.io/kubernetes/pkg/volume/vsphere_volume
+k8s.io/kubernetes/cmd/cloud-controller-manager/app/config
+k8s.io/kubernetes/cmd/cloud-controller-manager/app/options
+k8s.io/kubernetes/cmd/controller-manager/app
+k8s.io/kubernetes/pkg/controller/cloud
+k8s.io/kubernetes/pkg/controller/route
+k8s.io/kubernetes/pkg/controller/service
 k8s.io/kubernetes/cmd/kube-apiserver/app/options
 k8s.io/kubernetes/pkg/controller/serviceaccount
 k8s.io/kubernetes/pkg/generated/openapi
@ -1890,7 +1897,6 @@ k8s.io/kubernetes/pkg/kubeapiserver/server
 k8s.io/kubernetes/pkg/master/controller/crdregistration
 k8s.io/kubernetes/pkg/registry/cachesize
 k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap
-k8s.io/kubernetes/cmd/controller-manager/app
 k8s.io/kubernetes/cmd/kube-controller-manager/app/config
 k8s.io/kubernetes/cmd/kube-controller-manager/app/options
 k8s.io/kubernetes/pkg/controller
@ -1900,7 +1906,6 @@ k8s.io/kubernetes/pkg/controller/certificates/approver
 k8s.io/kubernetes/pkg/controller/certificates/cleaner
 k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher
 k8s.io/kubernetes/pkg/controller/certificates/signer
-k8s.io/kubernetes/pkg/controller/cloud
 k8s.io/kubernetes/pkg/controller/clusterroleaggregation
 k8s.io/kubernetes/pkg/controller/cronjob
 k8s.io/kubernetes/pkg/controller/daemon
@ -1920,8 +1925,6 @@ k8s.io/kubernetes/pkg/controller/podgc
 k8s.io/kubernetes/pkg/controller/replicaset
 k8s.io/kubernetes/pkg/controller/replication
 k8s.io/kubernetes/pkg/controller/resourcequota
-k8s.io/kubernetes/pkg/controller/route
-k8s.io/kubernetes/pkg/controller/service
 k8s.io/kubernetes/pkg/controller/statefulset
 k8s.io/kubernetes/pkg/controller/ttl
 k8s.io/kubernetes/pkg/controller/ttlafterfinished
@ -2214,9 +2217,13 @@ k8s.io/kubernetes/pkg/volume/util/recyclerclient
 k8s.io/kubernetes/pkg/volume/csi/csiv0
 k8s.io/kubernetes/pkg/volume/csi/nodeinfomanager
 k8s.io/kubernetes/pkg/volume/util/fsquota
+k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config
+k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/scheme
+k8s.io/kubernetes/cmd/cloud-controller-manager/app/apis/config/v1alpha1
+k8s.io/kubernetes/cmd/controller-manager/app/options
+k8s.io/kubernetes/pkg/controller/util/node
 k8s.io/kubernetes/pkg/registry/core/secret
 k8s.io/kubernetes/pkg/quota/v1
-k8s.io/kubernetes/cmd/controller-manager/app/options
 k8s.io/kubernetes/pkg/controller/apis/config/scheme
 k8s.io/kubernetes/pkg/controller/certificates/signer/config
 k8s.io/kubernetes/pkg/controller/daemon/config
@ -2239,7 +2246,6 @@ k8s.io/kubernetes/pkg/controller/ttlafterfinished/config
 k8s.io/kubernetes/pkg/controller/volume/attachdetach/config
 k8s.io/kubernetes/pkg/controller/service/config
 k8s.io/kubernetes/pkg/controller/certificates
-k8s.io/kubernetes/pkg/controller/util/node
 k8s.io/kubernetes/pkg/controller/daemon/util
 k8s.io/kubernetes/pkg/util/labels
 k8s.io/kubernetes/pkg/controller/deployment/util
@ -2380,6 +2386,7 @@ k8s.io/kubernetes/pkg/util/ebtables
 k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files
 k8s.io/kubernetes/pkg/volume/util/fsquota/common
 k8s.io/kubernetes/pkg/controller/apis/config/v1alpha1
+k8s.io/kubernetes/pkg/controller/service/config/v1alpha1
 k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations
 k8s.io/kubernetes/pkg/scheduler/algorithm/priorities
 k8s.io/kubernetes/pkg/scheduler/api/validation
@ -2432,7 +2439,6 @@ k8s.io/kubernetes/pkg/controller/podgc/config/v1alpha1
 k8s.io/kubernetes/pkg/controller/replicaset/config/v1alpha1
 k8s.io/kubernetes/pkg/controller/replication/config/v1alpha1
 k8s.io/kubernetes/pkg/controller/resourcequota/config/v1alpha1
-k8s.io/kubernetes/pkg/controller/service/config/v1alpha1
 k8s.io/kubernetes/pkg/controller/serviceaccount/config/v1alpha1
 k8s.io/kubernetes/pkg/controller/statefulset/config/v1alpha1
 k8s.io/kubernetes/pkg/controller/ttlafterfinished/config/v1alpha1