/* Copyright 2014 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controlplane import ( "context" "fmt" "net" "net/http" "reflect" "strconv" "time" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" apiserverinternalv1alpha1 "k8s.io/api/apiserverinternal/v1alpha1" appsv1 "k8s.io/api/apps/v1" authenticationv1 "k8s.io/api/authentication/v1" authenticationv1beta1 "k8s.io/api/authentication/v1beta1" authorizationapiv1 "k8s.io/api/authorization/v1" authorizationapiv1beta1 "k8s.io/api/authorization/v1beta1" autoscalingapiv1 "k8s.io/api/autoscaling/v1" autoscalingapiv2beta1 "k8s.io/api/autoscaling/v2beta1" autoscalingapiv2beta2 "k8s.io/api/autoscaling/v2beta2" batchapiv1 "k8s.io/api/batch/v1" batchapiv1beta1 "k8s.io/api/batch/v1beta1" certificatesapiv1 "k8s.io/api/certificates/v1" certificatesapiv1beta1 "k8s.io/api/certificates/v1beta1" coordinationapiv1 "k8s.io/api/coordination/v1" coordinationapiv1beta1 "k8s.io/api/coordination/v1beta1" apiv1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" discoveryv1beta1 "k8s.io/api/discovery/v1beta1" eventsv1 "k8s.io/api/events/v1" eventsv1beta1 "k8s.io/api/events/v1beta1" extensionsapiv1beta1 "k8s.io/api/extensions/v1beta1" flowcontrolv1alpha1 "k8s.io/api/flowcontrol/v1alpha1" networkingapiv1 "k8s.io/api/networking/v1" networkingapiv1beta1 "k8s.io/api/networking/v1beta1" nodev1 "k8s.io/api/node/v1" nodev1alpha1 "k8s.io/api/node/v1alpha1" nodev1beta1 "k8s.io/api/node/v1beta1" policyapiv1 "k8s.io/api/policy/v1" policyapiv1beta1 "k8s.io/api/policy/v1beta1" rbacv1 "k8s.io/api/rbac/v1" rbacv1alpha1 "k8s.io/api/rbac/v1alpha1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" schedulingapiv1 "k8s.io/api/scheduling/v1" schedulingv1alpha1 "k8s.io/api/scheduling/v1alpha1" schedulingapiv1beta1 "k8s.io/api/scheduling/v1beta1" storageapiv1 "k8s.io/api/storage/v1" storageapiv1alpha1 "k8s.io/api/storage/v1alpha1" storageapiv1beta1 "k8s.io/api/storage/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/clock" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/endpoints/discovery" apiserverfeatures "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/apiserver/pkg/server/healthz" serverstorage "k8s.io/apiserver/pkg/server/storage" storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" discoveryclient "k8s.io/client-go/kubernetes/typed/discovery/v1" "k8s.io/component-base/version" "k8s.io/component-helpers/apimachinery/lease" "k8s.io/klog/v2" api "k8s.io/kubernetes/pkg/apis/core" flowcontrolv1beta1 "k8s.io/kubernetes/pkg/apis/flowcontrol/v1beta1" "k8s.io/kubernetes/pkg/controlplane/controller/apiserverleasegc" "k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust" "k8s.io/kubernetes/pkg/controlplane/reconcilers" "k8s.io/kubernetes/pkg/controlplane/tunneler" kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" "k8s.io/kubernetes/pkg/routes" "k8s.io/kubernetes/pkg/serviceaccount" nodeutil "k8s.io/kubernetes/pkg/util/node" // RESTStorage installers admissionregistrationrest "k8s.io/kubernetes/pkg/registry/admissionregistration/rest" apiserverinternalrest "k8s.io/kubernetes/pkg/registry/apiserverinternal/rest" appsrest "k8s.io/kubernetes/pkg/registry/apps/rest" authenticationrest "k8s.io/kubernetes/pkg/registry/authentication/rest" authorizationrest "k8s.io/kubernetes/pkg/registry/authorization/rest" autoscalingrest "k8s.io/kubernetes/pkg/registry/autoscaling/rest" batchrest "k8s.io/kubernetes/pkg/registry/batch/rest" certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest" coordinationrest "k8s.io/kubernetes/pkg/registry/coordination/rest" corerest "k8s.io/kubernetes/pkg/registry/core/rest" discoveryrest "k8s.io/kubernetes/pkg/registry/discovery/rest" eventsrest "k8s.io/kubernetes/pkg/registry/events/rest" extensionsrest "k8s.io/kubernetes/pkg/registry/extensions/rest" flowcontrolrest "k8s.io/kubernetes/pkg/registry/flowcontrol/rest" networkingrest "k8s.io/kubernetes/pkg/registry/networking/rest" noderest "k8s.io/kubernetes/pkg/registry/node/rest" policyrest "k8s.io/kubernetes/pkg/registry/policy/rest" rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" schedulingrest "k8s.io/kubernetes/pkg/registry/scheduling/rest" storagerest "k8s.io/kubernetes/pkg/registry/storage/rest" ) const ( // DefaultEndpointReconcilerInterval is the default amount of time for how often the endpoints for // the kubernetes Service are reconciled. DefaultEndpointReconcilerInterval = 10 * time.Second // DefaultEndpointReconcilerTTL is the default TTL timeout for the storage layer DefaultEndpointReconcilerTTL = 15 * time.Second // IdentityLeaseComponentLabelKey is used to apply a component label to identity lease objects, indicating: // 1. the lease is an identity lease (different from leader election leases) // 2. which component owns this lease IdentityLeaseComponentLabelKey = "k8s.io/component" // KubeAPIServer defines variable used internally when referring to kube-apiserver component KubeAPIServer = "kube-apiserver" // KubeAPIServerIdentityLeaseLabelSelector selects kube-apiserver identity leases KubeAPIServerIdentityLeaseLabelSelector = IdentityLeaseComponentLabelKey + "=" + KubeAPIServer ) // ExtraConfig defines extra configuration for the master type ExtraConfig struct { ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo APIResourceConfigSource serverstorage.APIResourceConfigSource StorageFactory serverstorage.StorageFactory EndpointReconcilerConfig EndpointReconcilerConfig EventTTL time.Duration KubeletClientConfig kubeletclient.KubeletClientConfig // Used to start and monitor tunneling Tunneler tunneler.Tunneler EnableLogsSupport bool ProxyTransport http.RoundTripper // Values to build the IP addresses used by discovery // The range of IPs to be assigned to services with type=ClusterIP or greater ServiceIPRange net.IPNet // The IP address for the GenericAPIServer service (must be inside ServiceIPRange) APIServerServiceIP net.IP // dual stack services, the range represents an alternative IP range for service IP // must be of different family than primary (ServiceIPRange) SecondaryServiceIPRange net.IPNet // the secondary IP address the GenericAPIServer service (must be inside SecondaryServiceIPRange) SecondaryAPIServerServiceIP net.IP // Port for the apiserver service. APIServerServicePort int // TODO, we can probably group service related items into a substruct to make it easier to configure // the API server items and `Extra*` fields likely fit nicely together. // The range of ports to be assigned to services with type=NodePort or greater ServiceNodePortRange utilnet.PortRange // Additional ports to be exposed on the GenericAPIServer service // extraServicePorts is injectable in the event that more ports // (other than the default 443/tcp) are exposed on the GenericAPIServer // and those ports need to be load balanced by the GenericAPIServer // service because this pkg is linked by out-of-tree projects // like openshift which want to use the GenericAPIServer but also do // more stuff. ExtraServicePorts []apiv1.ServicePort // Additional ports to be exposed on the GenericAPIServer endpoints // Port names should align with ports defined in ExtraServicePorts ExtraEndpointPorts []apiv1.EndpointPort // If non-zero, the "kubernetes" services uses this port as NodePort. KubernetesServiceNodePort int // Number of masters running; all masters must be started with the // same value for this field. (Numbers > 1 currently untested.) MasterCount int // MasterEndpointReconcileTTL sets the time to live in seconds of an // endpoint record recorded by each master. The endpoints are checked at an // interval that is 2/3 of this value and this value defaults to 15s if // unset. In very large clusters, this value may be increased to reduce the // possibility that the master endpoint record expires (due to other load // on the etcd server) and causes masters to drop in and out of the // kubernetes service record. It is not recommended to set this value below // 15s. MasterEndpointReconcileTTL time.Duration // Selects which reconciler to use EndpointReconcilerType reconcilers.Type ServiceAccountIssuer serviceaccount.TokenGenerator ServiceAccountMaxExpiration time.Duration ExtendExpiration bool // ServiceAccountIssuerDiscovery ServiceAccountIssuerURL string ServiceAccountJWKSURI string ServiceAccountPublicKeys []interface{} VersionedInformers informers.SharedInformerFactory IdentityLeaseDurationSeconds int IdentityLeaseRenewIntervalSeconds int } // Config defines configuration for the master type Config struct { GenericConfig *genericapiserver.Config ExtraConfig ExtraConfig } type completedConfig struct { GenericConfig genericapiserver.CompletedConfig ExtraConfig *ExtraConfig } // CompletedConfig embeds a private pointer that cannot be instantiated outside of this package type CompletedConfig struct { *completedConfig } // EndpointReconcilerConfig holds the endpoint reconciler and endpoint reconciliation interval to be // used by the master. type EndpointReconcilerConfig struct { Reconciler reconcilers.EndpointReconciler Interval time.Duration } // Instance contains state for a Kubernetes cluster api server instance. type Instance struct { GenericAPIServer *genericapiserver.GenericAPIServer ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo } func (c *Config) createMasterCountReconciler() reconcilers.EndpointReconciler { endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) endpointSliceClient := discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient) return reconcilers.NewMasterCountEndpointReconciler(c.ExtraConfig.MasterCount, endpointsAdapter) } func (c *Config) createNoneReconciler() reconcilers.EndpointReconciler { return reconcilers.NewNoneEndpointReconciler() } func (c *Config) createLeaseReconciler() reconcilers.EndpointReconciler { endpointClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) endpointSliceClient := discoveryclient.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) endpointsAdapter := reconcilers.NewEndpointsAdapter(endpointClient, endpointSliceClient) ttl := c.ExtraConfig.MasterEndpointReconcileTTL config, err := c.ExtraConfig.StorageFactory.NewConfig(api.Resource("apiServerIPInfo")) if err != nil { klog.Fatalf("Error determining service IP ranges: %v", err) } leaseStorage, _, err := storagefactory.Create(*config, nil) if err != nil { klog.Fatalf("Error creating storage factory: %v", err) } masterLeases := reconcilers.NewLeases(leaseStorage, "/masterleases/", ttl) return reconcilers.NewLeaseEndpointReconciler(endpointsAdapter, masterLeases) } func (c *Config) createEndpointReconciler() reconcilers.EndpointReconciler { klog.Infof("Using reconciler: %v", c.ExtraConfig.EndpointReconcilerType) switch c.ExtraConfig.EndpointReconcilerType { // there are numerous test dependencies that depend on a default controller case "", reconcilers.MasterCountReconcilerType: return c.createMasterCountReconciler() case reconcilers.LeaseEndpointReconcilerType: return c.createLeaseReconciler() case reconcilers.NoneEndpointReconcilerType: return c.createNoneReconciler() default: klog.Fatalf("Reconciler not implemented: %v", c.ExtraConfig.EndpointReconcilerType) } return nil } // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. func (c *Config) Complete() CompletedConfig { cfg := completedConfig{ c.GenericConfig.Complete(c.ExtraConfig.VersionedInformers), &c.ExtraConfig, } serviceIPRange, apiServerServiceIP, err := ServiceIPRange(cfg.ExtraConfig.ServiceIPRange) if err != nil { klog.Fatalf("Error determining service IP ranges: %v", err) } if cfg.ExtraConfig.ServiceIPRange.IP == nil { cfg.ExtraConfig.ServiceIPRange = serviceIPRange } if cfg.ExtraConfig.APIServerServiceIP == nil { cfg.ExtraConfig.APIServerServiceIP = apiServerServiceIP } discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.GenericConfig.ExternalAddress} discoveryAddresses.CIDRRules = append(discoveryAddresses.CIDRRules, discovery.CIDRRule{IPRange: cfg.ExtraConfig.ServiceIPRange, Address: net.JoinHostPort(cfg.ExtraConfig.APIServerServiceIP.String(), strconv.Itoa(cfg.ExtraConfig.APIServerServicePort))}) cfg.GenericConfig.DiscoveryAddresses = discoveryAddresses if cfg.ExtraConfig.ServiceNodePortRange.Size == 0 { // TODO: Currently no way to specify an empty range (do we need to allow this?) // We should probably allow this for clouds that don't require NodePort to do load-balancing (GCE) // but then that breaks the strict nestedness of ServiceType. // Review post-v1 cfg.ExtraConfig.ServiceNodePortRange = kubeoptions.DefaultServiceNodePortRange klog.Infof("Node port range unspecified. Defaulting to %v.", cfg.ExtraConfig.ServiceNodePortRange) } if cfg.ExtraConfig.EndpointReconcilerConfig.Interval == 0 { cfg.ExtraConfig.EndpointReconcilerConfig.Interval = DefaultEndpointReconcilerInterval } if cfg.ExtraConfig.MasterEndpointReconcileTTL == 0 { cfg.ExtraConfig.MasterEndpointReconcileTTL = DefaultEndpointReconcilerTTL } if cfg.ExtraConfig.EndpointReconcilerConfig.Reconciler == nil { cfg.ExtraConfig.EndpointReconcilerConfig.Reconciler = c.createEndpointReconciler() } return CompletedConfig{&cfg} } // New returns a new instance of Master from the given config. // Certain config fields will be set to a default value if unset. // Certain config fields must be specified, including: // KubeletClientConfig func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) { if reflect.DeepEqual(c.ExtraConfig.KubeletClientConfig, kubeletclient.KubeletClientConfig{}) { return nil, fmt.Errorf("Master.New() called with empty config.KubeletClientConfig") } s, err := c.GenericConfig.New("kube-apiserver", delegationTarget) if err != nil { return nil, err } if c.ExtraConfig.EnableLogsSupport { routes.Logs{}.Install(s.Handler.GoRestfulContainer) } // Metadata and keys are expected to only change across restarts at present, // so we just marshal immediately and serve the cached JSON bytes. md, err := serviceaccount.NewOpenIDMetadata( c.ExtraConfig.ServiceAccountIssuerURL, c.ExtraConfig.ServiceAccountJWKSURI, c.GenericConfig.ExternalAddress, c.ExtraConfig.ServiceAccountPublicKeys, ) if err != nil { // If there was an error, skip installing the endpoints and log the // error, but continue on. We don't return the error because the // metadata responses require additional, backwards incompatible // validation of command-line options. msg := fmt.Sprintf("Could not construct pre-rendered responses for"+ " ServiceAccountIssuerDiscovery endpoints. Endpoints will not be"+ " enabled. Error: %v", err) if c.ExtraConfig.ServiceAccountIssuerURL != "" { // The user likely expects this feature to be enabled if issuer URL is // set and the feature gate is enabled. In the future, if there is no // longer a feature gate and issuer URL is not set, the user may not // expect this feature to be enabled. We log the former case as an Error // and the latter case as an Info. klog.Error(msg) } else { klog.Info(msg) } } else { routes.NewOpenIDMetadataServer(md.ConfigJSON, md.PublicKeysetJSON). Install(s.Handler.GoRestfulContainer) } m := &Instance{ GenericAPIServer: s, ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo, } // install legacy rest storage if c.ExtraConfig.APIResourceConfigSource.VersionEnabled(apiv1.SchemeGroupVersion) { legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{ StorageFactory: c.ExtraConfig.StorageFactory, ProxyTransport: c.ExtraConfig.ProxyTransport, KubeletClientConfig: c.ExtraConfig.KubeletClientConfig, EventTTL: c.ExtraConfig.EventTTL, ServiceIPRange: c.ExtraConfig.ServiceIPRange, SecondaryServiceIPRange: c.ExtraConfig.SecondaryServiceIPRange, ServiceNodePortRange: c.ExtraConfig.ServiceNodePortRange, LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig, ServiceAccountIssuer: c.ExtraConfig.ServiceAccountIssuer, ExtendExpiration: c.ExtraConfig.ExtendExpiration, ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration, APIAudiences: c.GenericConfig.Authentication.APIAudiences, } if err := m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter, legacyRESTStorageProvider); err != nil { return nil, err } } // The order here is preserved in discovery. // If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"), // the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer. // This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go // with specific priorities. // TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery // handlers that we have. restStorageProviders := []RESTStorageProvider{ apiserverinternalrest.StorageProvider{}, authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences}, authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver}, autoscalingrest.RESTStorageProvider{}, batchrest.RESTStorageProvider{}, certificatesrest.RESTStorageProvider{}, coordinationrest.RESTStorageProvider{}, discoveryrest.StorageProvider{}, extensionsrest.RESTStorageProvider{}, networkingrest.RESTStorageProvider{}, noderest.RESTStorageProvider{}, policyrest.RESTStorageProvider{}, rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer}, schedulingrest.RESTStorageProvider{}, storagerest.RESTStorageProvider{}, flowcontrolrest.RESTStorageProvider{}, // keep apps after extensions so legacy clients resolve the extensions versions of shared resource names. // See https://github.com/kubernetes/kubernetes/issues/42392 appsrest.StorageProvider{}, admissionregistrationrest.RESTStorageProvider{}, eventsrest.RESTStorageProvider{TTL: c.ExtraConfig.EventTTL}, } if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil { return nil, err } if c.ExtraConfig.Tunneler != nil { m.installTunneler(c.ExtraConfig.Tunneler, corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig).Nodes()) } m.GenericAPIServer.AddPostStartHookOrDie("start-cluster-authentication-info-controller", func(hookContext genericapiserver.PostStartHookContext) error { kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig) if err != nil { return err } controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, kubeClient) // prime values and start listeners if m.ClusterAuthenticationInfo.ClientCA != nil { if notifier, ok := m.ClusterAuthenticationInfo.ClientCA.(dynamiccertificates.Notifier); ok { notifier.AddListener(controller) } if controller, ok := m.ClusterAuthenticationInfo.ClientCA.(dynamiccertificates.ControllerRunner); ok { // runonce to be sure that we have a value. if err := controller.RunOnce(); err != nil { runtime.HandleError(err) } go controller.Run(1, hookContext.StopCh) } } if m.ClusterAuthenticationInfo.RequestHeaderCA != nil { if notifier, ok := m.ClusterAuthenticationInfo.RequestHeaderCA.(dynamiccertificates.Notifier); ok { notifier.AddListener(controller) } if controller, ok := m.ClusterAuthenticationInfo.RequestHeaderCA.(dynamiccertificates.ControllerRunner); ok { // runonce to be sure that we have a value. if err := controller.RunOnce(); err != nil { runtime.HandleError(err) } go controller.Run(1, hookContext.StopCh) } } go controller.Run(1, hookContext.StopCh) return nil }) if utilfeature.DefaultFeatureGate.Enabled(apiserverfeatures.APIServerIdentity) { m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-controller", func(hookContext genericapiserver.PostStartHookContext) error { kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig) if err != nil { return err } controller := lease.NewController( clock.RealClock{}, kubeClient, m.GenericAPIServer.APIServerID, int32(c.ExtraConfig.IdentityLeaseDurationSeconds), nil, time.Duration(c.ExtraConfig.IdentityLeaseRenewIntervalSeconds)*time.Second, metav1.NamespaceSystem, labelAPIServerHeartbeat) go controller.Run(wait.NeverStop) return nil }) m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-garbage-collector", func(hookContext genericapiserver.PostStartHookContext) error { kubeClient, err := kubernetes.NewForConfig(hookContext.LoopbackClientConfig) if err != nil { return err } go apiserverleasegc.NewAPIServerLeaseGC( kubeClient, time.Duration(c.ExtraConfig.IdentityLeaseDurationSeconds)*time.Second, metav1.NamespaceSystem, KubeAPIServerIdentityLeaseLabelSelector, ).Run(wait.NeverStop) return nil }) } return m, nil } func labelAPIServerHeartbeat(lease *coordinationapiv1.Lease) error { if lease.Labels == nil { lease.Labels = map[string]string{} } // This label indicates that kube-apiserver owns this identity lease object lease.Labels[IdentityLeaseComponentLabelKey] = KubeAPIServer return nil } // InstallLegacyAPI will install the legacy APIs for the restStorageProviders if they are enabled. func (m *Instance) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter, legacyRESTStorageProvider corerest.LegacyRESTStorageProvider) error { legacyRESTStorage, apiGroupInfo, err := legacyRESTStorageProvider.NewLegacyRESTStorage(restOptionsGetter) if err != nil { return fmt.Errorf("error building core storage: %v", err) } controllerName := "bootstrap-controller" coreClient := corev1client.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig) bootstrapController := c.NewBootstrapController(legacyRESTStorage, coreClient, coreClient, coreClient, coreClient.RESTClient()) m.GenericAPIServer.AddPostStartHookOrDie(controllerName, bootstrapController.PostStartHook) m.GenericAPIServer.AddPreShutdownHookOrDie(controllerName, bootstrapController.PreShutdownHook) if err := m.GenericAPIServer.InstallLegacyAPIGroup(genericapiserver.DefaultLegacyAPIPrefix, &apiGroupInfo); err != nil { return fmt.Errorf("error in registering group versions: %v", err) } return nil } func (m *Instance) installTunneler(nodeTunneler tunneler.Tunneler, nodeClient corev1client.NodeInterface) { nodeTunneler.Run(nodeAddressProvider{nodeClient}.externalAddresses) err := m.GenericAPIServer.AddHealthChecks(healthz.NamedCheck("SSH Tunnel Check", tunneler.TunnelSyncHealthChecker(nodeTunneler))) if err != nil { klog.Errorf("Failed adding ssh tunnel health check %v\n", err) } } // RESTStorageProvider is a factory type for REST storage. type RESTStorageProvider interface { GroupName() string NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, bool, error) } // InstallAPIs will install the APIs for the restStorageProviders if they are enabled. func (m *Instance) InstallAPIs(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter, restStorageProviders ...RESTStorageProvider) error { apiGroupsInfo := []*genericapiserver.APIGroupInfo{} // used later in the loop to filter the served resource by those that have expired. resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(version.Get()) if err != nil { return err } for _, restStorageBuilder := range restStorageProviders { groupName := restStorageBuilder.GroupName() if !apiResourceConfigSource.AnyVersionForGroupEnabled(groupName) { klog.V(1).Infof("Skipping disabled API group %q.", groupName) continue } apiGroupInfo, enabled, err := restStorageBuilder.NewRESTStorage(apiResourceConfigSource, restOptionsGetter) if err != nil { return fmt.Errorf("problem initializing API group %q : %v", groupName, err) } if !enabled { klog.Warningf("API group %q is not enabled, skipping.", groupName) continue } // Remove resources that serving kinds that are removed. // We do this here so that we don't accidentally serve versions without resources or openapi information that for kinds we don't serve. // This is a spot above the construction of individual storage handlers so that no sig accidentally forgets to check. resourceExpirationEvaluator.RemoveDeletedKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap) if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 { klog.V(1).Infof("Removing API group %v because it is time to stop serving it because it has no versions per APILifecycle.", groupName) continue } klog.V(1).Infof("Enabling API group %q.", groupName) if postHookProvider, ok := restStorageBuilder.(genericapiserver.PostStartHookProvider); ok { name, hook, err := postHookProvider.PostStartHook() if err != nil { klog.Fatalf("Error building PostStartHook: %v", err) } m.GenericAPIServer.AddPostStartHookOrDie(name, hook) } apiGroupsInfo = append(apiGroupsInfo, &apiGroupInfo) } if err := m.GenericAPIServer.InstallAPIGroups(apiGroupsInfo...); err != nil { return fmt.Errorf("error in registering group versions: %v", err) } return nil } type nodeAddressProvider struct { nodeClient corev1client.NodeInterface } func (n nodeAddressProvider) externalAddresses() ([]string, error) { preferredAddressTypes := []apiv1.NodeAddressType{ apiv1.NodeExternalIP, } nodes, err := n.nodeClient.List(context.TODO(), metav1.ListOptions{}) if err != nil { return nil, err } var matchErr error addrs := []string{} for ix := range nodes.Items { node := &nodes.Items[ix] addr, err := nodeutil.GetPreferredNodeAddress(node, preferredAddressTypes) if err != nil { if _, ok := err.(*nodeutil.NoMatchError); ok { matchErr = err continue } return nil, err } addrs = append(addrs, addr) } if len(addrs) == 0 && matchErr != nil { // We only return an error if we have items. // Currently we return empty list/no error if Items is empty. // We do this for backward compatibility reasons. return nil, matchErr } return addrs, nil } // DefaultAPIResourceConfigSource returns default configuration for an APIResource. func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig { ret := serverstorage.NewResourceConfig() // NOTE: GroupVersions listed here will be enabled by default. Don't put alpha versions in the list. ret.EnableVersions( admissionregistrationv1.SchemeGroupVersion, admissionregistrationv1beta1.SchemeGroupVersion, apiv1.SchemeGroupVersion, appsv1.SchemeGroupVersion, authenticationv1.SchemeGroupVersion, authenticationv1beta1.SchemeGroupVersion, authorizationapiv1.SchemeGroupVersion, authorizationapiv1beta1.SchemeGroupVersion, autoscalingapiv1.SchemeGroupVersion, autoscalingapiv2beta1.SchemeGroupVersion, autoscalingapiv2beta2.SchemeGroupVersion, batchapiv1.SchemeGroupVersion, batchapiv1beta1.SchemeGroupVersion, certificatesapiv1.SchemeGroupVersion, certificatesapiv1beta1.SchemeGroupVersion, coordinationapiv1.SchemeGroupVersion, coordinationapiv1beta1.SchemeGroupVersion, discoveryv1.SchemeGroupVersion, discoveryv1beta1.SchemeGroupVersion, eventsv1.SchemeGroupVersion, eventsv1beta1.SchemeGroupVersion, extensionsapiv1beta1.SchemeGroupVersion, networkingapiv1.SchemeGroupVersion, networkingapiv1beta1.SchemeGroupVersion, nodev1.SchemeGroupVersion, nodev1beta1.SchemeGroupVersion, policyapiv1.SchemeGroupVersion, policyapiv1beta1.SchemeGroupVersion, rbacv1.SchemeGroupVersion, rbacv1beta1.SchemeGroupVersion, storageapiv1.SchemeGroupVersion, storageapiv1beta1.SchemeGroupVersion, schedulingapiv1beta1.SchemeGroupVersion, schedulingapiv1.SchemeGroupVersion, flowcontrolv1beta1.SchemeGroupVersion, ) // enable non-deprecated beta resources in extensions/v1beta1 explicitly so we have a full list of what's possible to serve ret.EnableResources( extensionsapiv1beta1.SchemeGroupVersion.WithResource("ingresses"), ) // disable alpha versions explicitly so we have a full list of what's possible to serve ret.DisableVersions( apiserverinternalv1alpha1.SchemeGroupVersion, nodev1alpha1.SchemeGroupVersion, rbacv1alpha1.SchemeGroupVersion, schedulingv1alpha1.SchemeGroupVersion, storageapiv1alpha1.SchemeGroupVersion, flowcontrolv1alpha1.SchemeGroupVersion, ) return ret }