diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 69ec633ada..422a97fe37 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -425,7 +425,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) nodeConfig.AgentConfig.ClientCA = clientCAFile - nodeConfig.AgentConfig.ListenAddress = "0.0.0.0" nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController @@ -466,18 +465,20 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.Certificate = servingCert nodeConfig.AgentConfig.NodeIPs = nodeIPs - nodeIP, err := util.GetFirst4(nodeIPs) + nodeIP, listenAddress, _, err := util.GetFirstIP(nodeIPs) if err != nil { - return nil, errors.Wrap(err, "cannot configure IPv4 node-ip") + return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip") } nodeConfig.AgentConfig.NodeIP = nodeIP.String() + nodeConfig.AgentConfig.ListenAddress = listenAddress nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs // if configured, set NodeExternalIP to the first IPv4 address, for legacy clients + // unless only IPv6 address given if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 { - nodeExternalIP, err := util.GetFirst4(nodeConfig.AgentConfig.NodeExternalIPs) + nodeExternalIP, _, _, err := util.GetFirstIP(nodeConfig.AgentConfig.NodeExternalIPs) if err != nil { - return nil, errors.Wrap(err, "cannot configure IPv4 node-external-ip") + return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip") } nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String() } diff --git a/pkg/agent/run.go b/pkg/agent/run.go index cec764237f..da1ae0bbd9 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -58,8 +58,10 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { if err != nil { return errors.Wrap(err, "failed to validate node-ip") } + serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR) + clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR) - enableIPv6 := dualCluster || dualService || dualNode + enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6 || clusterIPv6 conntrackConfig, err := getConntrackConfig(nodeConfig) if err != nil { return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration") diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index b05b2823f1..f640dc3bcf 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -198,17 +198,17 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 { // ignoring the error here is fine since etcd will fall back to the interface's IPv4 address - serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP) + serverConfig.ControlConfig.PrivateIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP) } // if not set, try setting advertise-ip from agent node-external-ip if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 { - serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeExternalIP) + serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeExternalIP) } // if not set, try setting advertise-ip from agent node-ip if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 { - serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP) + serverConfig.ControlConfig.AdvertiseIP, _, _ = util.GetFirstString(cmds.AgentConfig.NodeIP) } // if we ended up with any advertise-ips, ensure they're added to the SAN list; @@ -226,14 +226,19 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont return err } serverConfig.ControlConfig.ServerNodeName = nodeName - serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "localhost", nodeName) + serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName) for _, ip := range nodeIPs { serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String()) } // configure ClusterIPRanges + _, _, IPv6only, _ := util.GetFirstIP(nodeIPs) if len(cmds.ServerConfig.ClusterCIDR) == 0 { - cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16") + clusterCIDR := "10.42.0.0/16" + if IPv6only { + clusterCIDR = "fd:42::/56" + } + cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR) } for _, cidr := range cmds.ServerConfig.ClusterCIDR { for _, v := range strings.Split(cidr, ",") { @@ -246,15 +251,20 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } // set ClusterIPRange to the first IPv4 block, for legacy clients - clusterIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ClusterIPRanges) + // unless only IPv6 range given + clusterIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ClusterIPRanges) if err != nil { - return errors.Wrap(err, "cannot configure IPv4 cluster-cidr") + return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr") } serverConfig.ControlConfig.ClusterIPRange = clusterIPRange // configure ServiceIPRanges if len(cmds.ServerConfig.ServiceCIDR) == 0 { - cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16") + serviceCIDR := "10.43.0.0/16" + if IPv6only { + serviceCIDR = "fd:43::/112" + } + cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR) } for _, cidr := range cmds.ServerConfig.ServiceCIDR { for _, v := range strings.Split(cidr, ",") { @@ -267,9 +277,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } // set ServiceIPRange to the first IPv4 block, for legacy clients - serviceIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ServiceIPRanges) + // unless only IPv6 range given + serviceIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ServiceIPRanges) if err != nil { - return errors.Wrap(err, "cannot configure IPv4 service-cidr") + return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr") } serverConfig.ControlConfig.ServiceIPRange = serviceIPRange @@ -287,7 +298,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont // If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10, // i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10 - // If there are no IPv4 ServiceCIDRs, an error will be raised. + // If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used. + // If neither of IPv4 or IPv6 are found an error is raised. if len(cmds.ServerConfig.ClusterDNS) == 0 { clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10) if err != nil { @@ -306,9 +318,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } } // Set ClusterDNS to the first IPv4 address, for legacy clients - clusterDNS, err := util.GetFirst4(serverConfig.ControlConfig.ClusterDNSs) + // unless only IPv6 range given + clusterDNS, _, _, err := util.GetFirstIP(serverConfig.ControlConfig.ClusterDNSs) if err != nil { - return errors.Wrap(err, "cannot configure IPv4 cluster-dns address") + return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address") } serverConfig.ControlConfig.ClusterDNS = clusterDNS } @@ -457,6 +470,9 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont ip := serverConfig.ControlConfig.BindAddress if ip == "" { ip = "127.0.0.1" + if IPv6only { + ip = "[::1]" + } } url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort) @@ -517,6 +533,16 @@ func validateNetworkConfiguration(serverConfig server.Config) error { return errors.New("dual-stack cluster-dns is not supported") } + IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(serverConfig.ControlConfig.ServiceIPRanges) + if IPv6OnlyService { + if serverConfig.ControlConfig.DisableNPC == false { + return errors.New("network policy enforcement is not compatible with IPv6 only operation; server must be restarted with --disable-network-policy") + } + if serverConfig.ControlConfig.FlannelBackend != config.FlannelBackendNone { + return errors.New("Flannel is not compatible with IPv6 only operation; server must be restarted with --flannel-backend=none") + } + } + return nil } diff --git a/pkg/daemons/agent/agent_linux.go b/pkg/daemons/agent/agent_linux.go index 4a04e57549..0d9b1863f4 100644 --- a/pkg/daemons/agent/agent_linux.go +++ b/pkg/daemons/agent/agent_linux.go @@ -39,9 +39,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) { } func kubeProxyArgs(cfg *config.Agent) map[string]string { + bindAddress := "127.0.0.1" + _, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP}) + if IPv6only { + bindAddress = "::1" + } argsMap := map[string]string{ "proxy-mode": "iptables", - "healthz-bind-address": "127.0.0.1", + "healthz-bind-address": bindAddress, "kubeconfig": cfg.KubeConfigKubeProxy, "cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs), "conntrack-max-per-core": "0", @@ -55,8 +60,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string { } func kubeletArgs(cfg *config.Agent) map[string]string { + bindAddress := "127.0.0.1" + _, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP}) + if IPv6only { + bindAddress = "::1" + } argsMap := map[string]string{ - "healthz-bind-address": "127.0.0.1", + "healthz-bind-address": bindAddress, "read-only-port": "0", "cluster-domain": cfg.ClusterDomain, "kubeconfig": cfg.KubeConfigKubelet, diff --git a/pkg/daemons/agent/agent_windows.go b/pkg/daemons/agent/agent_windows.go index 0af9435f61..ba2542ffcd 100644 --- a/pkg/daemons/agent/agent_windows.go +++ b/pkg/daemons/agent/agent_windows.go @@ -27,9 +27,14 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) { } func kubeProxyArgs(cfg *config.Agent) map[string]string { + bindAddress := "127.0.0.1" + _, IPv6only, _ := util.GetFirstString(cfg.NodeIP) + if IPv6only { + bindAddress = "::1" + } argsMap := map[string]string{ "proxy-mode": "kernelspace", - "healthz-bind-address": "127.0.0.1", + "healthz-bind-address": bindAddress, "kubeconfig": cfg.KubeConfigKubeProxy, "cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs), } @@ -45,8 +50,13 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string { } func kubeletArgs(cfg *config.Agent) map[string]string { + bindAddress := "127.0.0.1" + _, IPv6only, _ := util.GetFirstString(cfg.NodeIP) + if IPv6only { + bindAddress = "::1" + } argsMap := map[string]string{ - "healthz-bind-address": "127.0.0.1", + "healthz-bind-address": bindAddress, "read-only-port": "0", "cluster-domain": cfg.ClusterDomain, "kubeconfig": cfg.KubeConfigKubelet, diff --git a/pkg/util/net.go b/pkg/util/net.go index 62e14cd9b8..804a33b917 100644 --- a/pkg/util/net.go +++ b/pkg/util/net.go @@ -80,6 +80,46 @@ func JoinIP4Nets(elems []*net.IPNet) string { return strings.Join(strs, ",") } +// GetFirst6 returns the first IPv6 address from the list of IP addresses. +// If no IPv6 addresses are found, an error is raised. +func GetFirst6(elems []net.IP) (net.IP, error) { + for _, elem := range elems { + if elem == nil || elem.To16() == nil { + continue + } + return elem, nil + } + return nil, errors.New("no IPv6 address found") +} + +// GetFirst6Net returns the first IPv4 network from the list of IP networks. +// If no IPv6 addresses are found, an error is raised. +func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) { + for _, elem := range elems { + if elem == nil || elem.IP.To16() == nil { + continue + } + return elem, nil + } + return nil, errors.New("no IPv6 CIDRs found") +} + +// GetFirst6String returns the first IPv6 address from a list of IP address strings. +// If no IPv6 addresses are found, an error is raised. +func GetFirst6String(elems []string) (string, error) { + ips := []net.IP{} + for _, elem := range elems { + for _, v := range strings.Split(elem, ",") { + ips = append(ips, net.ParseIP(v)) + } + } + ip, err := GetFirst6(ips) + if err != nil { + return "", err + } + return ip.String(), nil +} + // JoinIP6Nets stringifies and joins a list of IPv6 networks with commas. func JoinIP6Nets(elems []*net.IPNet) string { var strs []string @@ -141,3 +181,78 @@ func ParseStringSliceToIPs(s cli.StringSlice) ([]net.IP, error) { return ips, nil } + +// GetFirstIP returns the first IPv4 address from the list of IP addresses. +// If no IPv4 addresses are found, returns the first IPv6 address +// if neither of IPv4 or IPv6 are found an error is raised. +// Additionally matching listen address and IP version is returned. +func GetFirstIP(nodeIPs []net.IP) (net.IP, string, bool, error) { + nodeIP, err := GetFirst4(nodeIPs) + ListenAddress := "0.0.0.0" + IPv6only := false + if err != nil { + nodeIP, err = GetFirst6(nodeIPs) + if err != nil { + return nil, "", false, err + } + ListenAddress = "::" + IPv6only = true + } + return nodeIP, ListenAddress, IPv6only, nil +} + +// GetFirstNet returns the first IPv4 network from the list of IP networks. +// If no IPv4 addresses are found, returns the first IPv6 address +// if neither of IPv4 or IPv6 are found an error is raised. +func GetFirstNet(elems []*net.IPNet) (*net.IPNet, error) { + serviceIPRange, err := GetFirst4Net(elems) + if err != nil { + serviceIPRange, err = GetFirst6Net(elems) + if err != nil { + return nil, err + } + } + return serviceIPRange, nil +} + +// GetFirstString returns the first IP4 address from a list of IP address strings. +// If no IPv4 addresses are found, returns the first IPv6 address +// if neither of IPv4 or IPv6 are found an error is raised. +func GetFirstString(elems []string) (string, bool, error) { + ip, err := GetFirst4String(elems) + IPv6only := false + if err != nil { + ip, err = GetFirst6String(elems) + if err != nil { + return "", false, err + } + IPv6only = true + } + return ip, IPv6only, nil +} + +// IsIPv6OnlyCIDRs returns if +// - all are valid cidrs +// - at least one cidr from v6 family is found +// - v4 family cidr is not found +func IsIPv6OnlyCIDRs(cidrs []*net.IPNet) (bool, error) { + v4Found := false + v6Found := false + for _, cidr := range cidrs { + if cidr == nil { + return false, fmt.Errorf("cidr %v is invalid", cidr) + } + + if v4Found && v6Found { + continue + } + + if cidr.IP != nil && cidr.IP.To4() == nil { + v6Found = true + continue + } + v4Found = true + } + + return !v4Found && v6Found, nil +}